From fc5eae45bcc0e796dc2d89b42dae64fc8ec72109 Mon Sep 17 00:00:00 2001 From: Eleven Date: Tue, 1 Feb 2022 20:48:53 -0600 Subject: [PATCH 1/4] refactor: flat structure Signed-off-by: Eleven <53446819+voidswordQQJ@users.noreply.github.com> --- .gitignore | 4 +- .idea/codestream.xml | 6 + .idea/dbnavigator.xml | 459 ++++++++++++++++++++++++++++ .idea/planetside.iml | 10 +- README.md | 11 +- config/.gitignore | 2 - config/config.json | 9 - config/logging.json | 32 -- controller/mysql_controller.py | 60 ---- ps2cpcdata/__init__.py | 8 + ps2cpcdata/__main__.py | 56 ++++ ps2cpcdata/config/config.json | 13 + ps2cpcdata/database.py | 64 ++++ ps2cpcdata/version.py | 5 + requirements.txt | 3 + setup.py | 17 ++ test/.gitkeeper | 0 test/config/.gitkeeper | 0 log/.gitkeeper => tests/__init__.py | 0 update_database.py | 91 ------ 20 files changed, 649 insertions(+), 201 deletions(-) create mode 100644 .idea/codestream.xml create mode 100644 .idea/dbnavigator.xml delete mode 100644 config/.gitignore delete mode 100644 config/config.json delete mode 100644 config/logging.json delete mode 100644 controller/mysql_controller.py create mode 100644 ps2cpcdata/__init__.py create mode 100644 ps2cpcdata/__main__.py create mode 100644 ps2cpcdata/config/config.json create mode 100644 ps2cpcdata/database.py create mode 100644 ps2cpcdata/version.py create mode 100644 requirements.txt create mode 100644 setup.py delete mode 100644 test/.gitkeeper delete mode 100644 test/config/.gitkeeper rename log/.gitkeeper => tests/__init__.py (100%) delete mode 100644 update_database.py diff --git a/.gitignore b/.gitignore index 103bc66..4199357 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1 @@ -# Ignore log files -/log/ -/test/ +/tests/config/config.json diff --git a/.idea/codestream.xml b/.idea/codestream.xml new file mode 100644 index 0000000..6a4410a --- /dev/null +++ b/.idea/codestream.xml @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/.idea/dbnavigator.xml b/.idea/dbnavigator.xml new file mode 100644 index 0000000..1c4a5a7 --- /dev/null +++ b/.idea/dbnavigator.xml @@ -0,0 +1,459 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/planetside.iml b/.idea/planetside.iml index 5ffb9af..f06a9d6 100644 --- a/.idea/planetside.iml +++ b/.idea/planetside.iml @@ -9,6 +9,14 @@ - + + \ No newline at end of file diff --git a/README.md b/README.md index f40574c..4e625e9 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,10 @@ Get game events and sync to database. "database": "example", "user": "user", "password": "password" + }, + "planetside2": { + "api": "wss://push.planetside2.com/streaming?environment=ps2&service-id=s: ...", + "subscription": "{\"service\":\"event\", ... , \"logicalAndCharactersWithWorlds\":true}" } } ``` @@ -23,12 +27,13 @@ Get game events and sync to database. 2. You will need following third-party dependencies to run the application. ```requirements.txt -pymysql >= 1.0.2 -websockets >= 10.1 +setuptools==57.0.0 +mysql-connector-python==8.0.28 +websockets==10.1 ``` 3. Use the command below to start. ```shell -python -m update_database.py +python -m ps2cpcdata ``` diff --git a/config/.gitignore b/config/.gitignore deleted file mode 100644 index b6d86fd..0000000 --- a/config/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# Ignore database config update -/config.json diff --git a/config/config.json b/config/config.json deleted file mode 100644 index 55b56cc..0000000 --- a/config/config.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "database": { - "host": "localhost", - "port": 3306, - "database": "example", - "user": "user", - "password": "password" - } -} \ No newline at end of file diff --git a/config/logging.json b/config/logging.json deleted file mode 100644 index 9ccd539..0000000 --- a/config/logging.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "version": 1, - "disable_existing_loggers": false, - "root": { - "level": "DEBUG", - "handlers": [ - "monitor", - "runtime" - ] - }, - "handlers": { - "monitor": { - "class": "logging.StreamHandler", - "formatter": "default", - "level": "DEBUG", - "stream": "ext://sys.stdout" - }, - "runtime": { - "class": "logging.handlers.TimedRotatingFileHandler", - "formatter": "default", - "level": "INFO", - "filename": "log/runtime.log", - "when": "midnight", - "backupCount": "3" - } - }, - "formatters": { - "default": { - "format": "%(asctime)s %(levelname)-8s %(message)s" - } - } -} \ No newline at end of file diff --git a/controller/mysql_controller.py b/controller/mysql_controller.py deleted file mode 100644 index d8ecccb..0000000 --- a/controller/mysql_controller.py +++ /dev/null @@ -1,60 +0,0 @@ -import json - -import pymysql - - -class Mysql(object): - """数据库相关操作 - - """ - - def __init__(self): - with open("config/config.json") as config: - database = json.load(config)["database"] - - self.conn = pymysql.connect(host=database["host"], port=database["port"], db=database["database"], - user=database["user"], password=database["password"]) - - async def update_event_in_background(self, payload): - pass - - -class DeathEventHandler(Mysql): - """死亡事件 - - """ - - async def update_event_in_background(self, payload): - """击杀数据库更新 - - :param payload: Websocket 订阅数据，字典类。 - """ - with self.conn.cursor() as cursor: - sql = "INSERT INTO ps2_death (attacker_character_id, attacker_fire_mode_id, attacker_loadout_id, " \ - "attacker_vehicle_id, attacker_weapon_id, character_id, character_loadout_id, is_headshot, " \ - "world_id, zone_id) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)" - cursor.execute(sql, (payload["attacker_character_id"], payload["attacker_fire_mode_id"], - payload["attacker_loadout_id"], payload["attacker_vehicle_id"], - payload["attacker_weapon_id"], payload["character_id"], - payload["character_loadout_id"], payload["is_headshot"], - payload["world_id"], payload["zone_id"])) - self.conn.commit() - - -class AlertEventHandler(Mysql): - """警报事件 - - """ - - async def update_event_in_background(self, payload): - """警报数据库更新 - - :param payload: Websocket 订阅数据，字典类。 - """ - with self.conn.cursor() as cursor: - sql = "INSERT INTO ps2_jingbao (faction_vs, faction_tr, faction_nc, world_id, zone_id, " \ - "metagame_event_id, metagame_event_state) VALUES (%s, %s, %s, %s, %s, %s, %s)" - cursor.execute(sql, (payload["faction_vs"], payload["faction_tr"], payload["faction_nc"], - payload["world_id"], payload["zone_id"], payload["metagame_event_id"], - payload["metagame_event_state"])) - self.conn.commit() diff --git a/ps2cpcdata/__init__.py b/ps2cpcdata/__init__.py new file mode 100644 index 0000000..c9d7d72 --- /dev/null +++ b/ps2cpcdata/__init__.py @@ -0,0 +1,8 @@ +"""The program initializes the configuration file and executes the main program.""" +import json +import logging + +logger = logging.getLogger(__name__) + +with open("ps2cpcdata/config/config.json", "r") as config: + __config__ = json.load(config) diff --git a/ps2cpcdata/__main__.py b/ps2cpcdata/__main__.py new file mode 100644 index 0000000..8d07f33 --- /dev/null +++ b/ps2cpcdata/__main__.py @@ -0,0 +1,56 @@ +"""This module implements the core developer interface for ps2cpcdata.""" +import asyncio +import json +import logging +import sys + +import websockets + +from ps2cpcdata import __config__ +from ps2cpcdata import database + +logging.basicConfig(stream=sys.stderr, level="NOTSET") +logger = logging.getLogger(__name__) + + +class PS2CPCData(object): + def __init__(self): + self.planetside = __config__["planetside2"] + self.api = self.planetside["api"] + self.subscription = self.planetside["subscription"] + self.database = database.Mysql() + + async def connect(self): + async with websockets.connect(self.api, ping_timeout=None) as ws: + await ws.send(self.subscription) + while True: + message = await ws.recv() + data = json.loads(message) + await self.update_database(data) + + async def update_database(self, data): + try: + payload = data["payload"] + except KeyError: + return + + if "Death" in payload.values(): + await self.database.death(payload) + + elif "MetagameEvent" in payload.values(): + await self.database.alert(payload) + + +ps2cpc_data = PS2CPCData() + +while True: + try: + asyncio.run(ps2cpc_data.connect()) + + except KeyboardInterrupt: + logger.info("The program was closed by the user.") + break + + except websockets.WebSocketException: + logger.warning("Connection failed, try to reconnect.") + continue diff --git a/ps2cpcdata/config/config.json b/ps2cpcdata/config/config.json new file mode 100644 index 0000000..2ea4a5c --- /dev/null +++ b/ps2cpcdata/config/config.json @@ -0,0 +1,13 @@ +{ + "database": { + "host": "localhost", + "port": 3306, + "database": "example", + "user": "user", + "password": "password" + }, + "planetside2": { + "api": "wss://push.planetside2.com/streaming?environment=ps2&service-id=s:yinxue", + "subscription": "{\"service\":\"event\",\"action\":\"subscribe\",\"characters\":[\"all\"],\"eventNames\":[\"Death\", \"MetagameEvent\"],\"worlds\":[\"1\", \"10\", \"13\", \"17\", \"40\"], \"logicalAndCharactersWithWorlds\":true}" + } +} \ No newline at end of file diff --git a/ps2cpcdata/database.py b/ps2cpcdata/database.py new file mode 100644 index 0000000..b3618c3 --- /dev/null +++ b/ps2cpcdata/database.py @@ -0,0 +1,64 @@ +"""This module contains all database operation.""" +import logging + +import mysql.connector + +from ps2cpcdata import __config__ + +logger = logging.getLogger(__name__) + + +class Mysql(object): + """Mysql database operation.""" + + def __init__(self): + self.database = __config__["database"] + self.conn = mysql.connector.connect(**self.database) + + async def _update(self, insert_event, event): + cursor = None + try: + cursor = self.conn.cursor() + cursor.execute(insert_event, event) + except mysql.connector.errors.Error as err: + logger.exception(err) + else: + self.conn.commit() + finally: + cursor.close() + + async def death(self, payload): + """Update death event. + + Args: + payload: Death event. + + Returns: None. + + """ + insert_death = "INSERT INTO ps2_death (attacker_character_id, attacker_fire_mode_id, attacker_loadout_id, " \ + "attacker_vehicle_id, attacker_weapon_id, character_id, character_loadout_id, is_headshot, " \ + "world_id, zone_id) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)" + death = (payload["attacker_character_id"], payload["attacker_fire_mode_id"], payload["attacker_loadout_id"], + payload["attacker_vehicle_id"], payload["attacker_weapon_id"], payload["character_id"], + payload["character_loadout_id"], payload["is_headshot"], payload["world_id"], payload["zone_id"]) + await self._update(insert_death, death) + + async def alert(self, payload): + """Update death event. + + Args: + payload: Alert event. + + Returns: None. + + """ + insert_alert = "INSERT INTO ps2_jingbao (faction_vs, faction_tr, faction_nc, world_id, zone_id, " \ + "metagame_event_id, metagame_event_state) VALUES (%s, %s, %s, %s, %s, %s, %s)" + alert = ((payload["faction_vs"], payload["faction_tr"], payload["faction_nc"], payload["world_id"], + payload["zone_id"], payload["metagame_event_id"], payload["metagame_event_state"])) + await self._update(insert_alert, alert) + + +if __name__ == '__main__': + pass diff --git a/ps2cpcdata/version.py b/ps2cpcdata/version.py new file mode 100644 index 0000000..85b6e28 --- /dev/null +++ b/ps2cpcdata/version.py @@ -0,0 +1,5 @@ +"""Just a version information.""" +__version__ = "1.0.0" + +if __name__ == '__main__': + print(__version__) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..64c1efc --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +setuptools==57.0.0 +mysql-connector-python==8.0.28 +websockets==10.1 \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..a7119e5 --- /dev/null +++ b/setup.py @@ -0,0 +1,17 @@ +from setuptools import setup + +from ps2cpcdata.version import __version__ + +setup( + name="ps2cpcdata", + version=__version__, + description="This is the first stable release of the package.", + author="ecss11", + maintainer="ecss11", + url="https://github.com/PlanetSide2-CPC/PS2-DatabaseSync", + packages=["ps2cpcdata"], + install_requires=[ + "mysql-connector-python==8.0.28", + "websockets==10.1" + ] +) diff --git a/test/.gitkeeper b/test/.gitkeeper deleted file mode 100644 index e69de29..0000000 diff --git a/test/config/.gitkeeper b/test/config/.gitkeeper deleted file mode 100644 index e69de29..0000000 diff --git a/log/.gitkeeper b/tests/__init__.py similarity index 100% rename from log/.gitkeeper rename to tests/__init__.py diff --git a/update_database.py b/update_database.py deleted file mode 100644 index c3c6f3d..0000000 --- a/update_database.py +++ /dev/null @@ -1,91 +0,0 @@ -import asyncio -import json -import logging.config -import logging.handlers -import os - -import websockets - -from controller import mysql_controller - - -class Subscription(object): - """同步订阅事件至数据库，使用 Websockets - - """ - - def __init__(self): - # Websocket API 订阅内容，http://census.daybreakgames.com/ - self.ps_api = "wss://push.planetside2.com/streaming?environment=ps2&service-id=s:yinxue" - self.subscribe = '{"service":"event","action":"subscribe","characters":["all"],"eventNames":["Death", ' \ - '"MetagameEvent"],"worlds":["1", "10", "13", "17", "40"],' \ - '"logicalAndCharactersWithWorlds":true} ' - self.death_handler = mysql_controller.DeathEventHandler() - self.alert_handler = mysql_controller.AlertEventHandler() - - async def connect_ps_api(self): - """连接行星边际 API 接口，并调用更新方法同步数据 - - """ - async with websockets.connect(self.ps_api, ping_timeout=None) as ws: - logger.info("Connection established.") - await ws.send(self.subscribe) - while True: - message = await ws.recv() - data: dict = json.loads(message) - - await self.sync_data_to_database(data) - - async def sync_data_to_database(self, data): - """同步数据至数据库 - - :param data: API 返回数据 - """ - is_subscription_event = True and data.get("service") == "event" and data.get("type") == "serviceMessage" - - if is_subscription_event: - await self.select_event_handler(data) - - async def select_event_handler(self, data): - """匹配事件对应的数据库操作 - - :param data: API 返回数据 - """ - payload: dict = data["payload"] - - if payload.get("event_name") == "Death": - await self.death_handler.update_event_in_background(payload) - - elif payload.get("event_name") == "MetagameEvent": - await self.alert_handler.update_event_in_background(payload) - - -def setup_logging(): - with open("config/logging.json", "r") as logging_config_file: - logging_config = json.load(logging_config_file) - - logging.config.dictConfig(logging_config) - - return logging.getLogger() - - -logger = setup_logging() - - -if __name__ == '__main__': - current_file_path = os.path.dirname(__file__) - os.chdir(current_file_path) - - synchronize = Subscription() - - while True: - try: - asyncio.run(synchronize.connect_ps_api()) - - except KeyboardInterrupt: - logger.info("The program was closed by the user.") - break - - except websockets.WebSocketException: - logger.warning("Connection failed, try to reconnect.") - continue From 2553133eb7b6234188c426f4259fcd3f23f3ae4c Mon Sep 17 00:00:00 2001 From: Eleven <53446819+voidswordQQJ@users.noreply.github.com> Date: Tue, 1 Feb 2022 19:17:05 -0800 Subject: [PATCH 2/4] Update version.py --- ps2cpcdata/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ps2cpcdata/version.py b/ps2cpcdata/version.py index 85b6e28..5921c23 100644 --- a/ps2cpcdata/version.py +++ b/ps2cpcdata/version.py @@ -1,5 +1,5 @@ """Just a version information.""" -__version__ = "1.0.0" +__version__ = "2.0.0" if __name__ == '__main__': print(__version__) From aec47b4479d1b00ba6d2fcdacfcc5b9ddeb8899d Mon Sep 17 00:00:00 2001 From: Eleven <53446819+voidswordQQJ@users.noreply.github.com> Date: Tue, 1 Feb 2022 19:18:24 -0800 Subject: [PATCH 3/4] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a7119e5..9c189e2 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name="ps2cpcdata", version=__version__, - description="This is the first stable release of the package.", + description="A tool for updating data in real time for the database.", author="ecss11", maintainer="ecss11", url="https://github.com/PlanetSide2-CPC/PS2-DatabaseSync", From d59bd1217385e5abf9fbd041d671653ffb791852 Mon Sep 17 00:00:00 2001 From: Eleven <53446819+voidswordQQJ@users.noreply.github.com> Date: Wed, 2 Feb 2022 16:04:36 -0600 Subject: [PATCH 4/4] docs: new format and description Signed-off-by: Eleven <53446819+voidswordQQJ@users.noreply.github.com> --- README.md | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4e625e9..1b89057 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,29 @@ # PS2-DatabaseSync +![GitHub release (latest by date)](https://img.shields.io/github/v/release/PlanetSide2-CPC/PS2-DatabaseSync) +[![GitHub issues](https://img.shields.io/github/issues/PlanetSide2-CPC/PS2-DatabaseSync)](https://github.com/PlanetSide2-CPC/PS2-DatabaseSync/issues) +[![GitHub license](https://img.shields.io/github/license/PlanetSide2-CPC/PS2-DatabaseSync)](https://github.com/PlanetSide2-CPC/PS2-DatabaseSync/blob/master/LICENSE) + > A tool based on the Planetside 2 API. Get game events and sync to database. -## How to use +## Documentation + +If you just want to run the program, then the [quickstart](README.md#Quickstart) guide is all you need. + +See https://github.com/PlanetSide2-CPC/PS2-DatabaseSync/pull/3 for how to add a new handler. + +## Features + +- Support custom subscription events. +- Only use async sync, no threading. + +## Quickstart + +This guide is only meant to cover the most basic usage of the library. + +### Installation 1. First you need a `config.json` file in config, for example: @@ -32,8 +51,20 @@ mysql-connector-python==8.0.28 websockets==10.1 ``` -3. Use the command below to start. +Or it can be installed automatically using setup.py . + +```shell +python -m pip install . +``` + +### Run the program + +Use the command below to start. ```shell python -m ps2cpcdata ``` + +## Contributing + +Issues Tracker: https://github.com/PlanetSide2-CPC/PS2-DatabaseSync/issues