From e162948cb8333e23350167865e0f9628b62b3145 Mon Sep 17 00:00:00 2001 From: DasMoorhuhn Date: Wed, 3 Jul 2024 03:59:38 +0200 Subject: [PATCH 1/8] add gatewayfinder and root api endpoint --- python/docker_entrypoint.sh | 1 + python/requierements.txt | 3 +- python/src/api_endpoints.py | 24 +++++++++++++++- python/src/find_gateways.py | 55 +++++++++++++++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 python/src/find_gateways.py diff --git a/python/docker_entrypoint.sh b/python/docker_entrypoint.sh index aeb780b..06a4647 100644 --- a/python/docker_entrypoint.sh +++ b/python/docker_entrypoint.sh @@ -5,6 +5,7 @@ env > .env if [ "$API" = true ]; then python3.12 api_endpoints.py & sleep 1 + python3.12 find_gateways.py & fi python3.12 main.py \ No newline at end of file diff --git a/python/requierements.txt b/python/requierements.txt index b7ed443..17e3ae8 100644 --- a/python/requierements.txt +++ b/python/requierements.txt @@ -3,4 +3,5 @@ pyyaml bs4 requests flask -flask_cors \ No newline at end of file +flask_cors +FindMyIP \ No newline at end of file diff --git a/python/src/api_endpoints.py b/python/src/api_endpoints.py index 6448f3b..0584760 100644 --- a/python/src/api_endpoints.py +++ b/python/src/api_endpoints.py @@ -17,6 +17,18 @@ class API: self.app.config['CORS_HEADERS'] = 'Content-Type' # --------Static Routes------- + @self.app.route('/') + @cross_origin() + def serve_root(): + root_dict = { + "version": "24-07-03", + "mode": 1, + "info": { + "files_size_sum": self.get_file_size() + } + } + return jsonify(root_dict) + @self.app.route('/charts') @cross_origin() def serve_index(): @@ -31,9 +43,19 @@ class API: @self.app.route('/json/') @cross_origin() def serve_json(path): - workdir, filename = os.path.split(os.path.abspath(__file__)) + return send_from_directory(f'{workdir}/data', path) + def get_file_size(self): + workdir, filename = os.path.split(os.path.abspath(__file__)) + files = os.listdir(f'{workdir}/data') + sizes = 0.0 + for file in files: + if file.endswith('.json'): + sizes += round(int(os.path.getsize(f'{workdir}/data/{file}')), 2) + + return sizes + api = API() api.app.run(host='0.0.0.0', port=8000) diff --git a/python/src/find_gateways.py b/python/src/find_gateways.py new file mode 100644 index 0000000..e89ec39 --- /dev/null +++ b/python/src/find_gateways.py @@ -0,0 +1,55 @@ +from time import sleep +import socket +import FindMyIP +import ipaddress +import threading +import requests + +max_threads = 50 +final = {} + + +def check_port(ip, port): + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # TCP + # sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # UDP + socket.setdefaulttimeout(2.0) # seconds (float) + result = sock.connect_ex((ip, port)) + if result == 0: + # print("Port is open") + final[ip] = "OPEN" + else: + # print("Port is closed/filtered") + final[ip] = "CLOSED" + sock.close() + except: + final[ip] = "EXCEPTION" + + +port = 8000 +local_ip = FindMyIP.internal() +local_ip = local_ip.split('.')[:-1] +local_ip.append("0") +local_ip = '.'.join(local_ip) + +print(f"Scan on {local_ip}/24 for port {port}") + +for ip in ipaddress.IPv4Network(f'{local_ip}/24'): + threading.Thread(target=check_port, args=[str(ip), port]).start() + # sleep(0.1) + + # limit the number of threads. + while threading.active_count() > max_threads: + sleep(1) + +sorted_ips = dict(sorted(final.items(), key=lambda item: tuple(map(int, item[0].split('.'))))) + +for ip, state in sorted_ips.items(): + if state == "OPEN": + try: + response = requests.get(f'http://{ip}:{port}/') + if response.status_code == 200: + print(ip, state) + print(response.text) + except:pass + From d7b3303e50e7ded3757e04933ae25f87158bc4f0 Mon Sep 17 00:00:00 2001 From: DasMoorhuhn Date: Thu, 4 Jul 2024 01:35:38 +0200 Subject: [PATCH 2/8] Add MQTT and gateway discovery --- Dockerfile | 3 ++ python/docker_entrypoint.sh | 1 - python/requierements.txt | 3 +- python/src/api_endpoints.py | 42 +++++++++++---------- python/src/check_update.py | 74 +++++++++++++++++++++++++++++++++++++ python/src/find_gateways.py | 59 +++++++++++++++-------------- python/src/main.py | 7 ++++ python/src/mqtt.py | 16 +++++++- run_gateway.sh | 23 +++++++++++- 9 files changed, 174 insertions(+), 54 deletions(-) create mode 100644 python/src/check_update.py diff --git a/Dockerfile b/Dockerfile index bf958e5..7fb1781 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,6 +45,9 @@ RUN apk add --no-cache sudo bluez tzdata ENV TZ=Europe/Berlin ENV DOCKER=true ENV API=false +ENV NAME=ATC_MiThermometer_Gateway +ENV VERSION=24-07-03 +ENV MODE=1 # Copy pips from the pip build stage COPY --from=pip_build_stage /usr/local/lib/python3.12/site-packages /usr/local/lib/python3.12/site-packages diff --git a/python/docker_entrypoint.sh b/python/docker_entrypoint.sh index 06a4647..aeb780b 100644 --- a/python/docker_entrypoint.sh +++ b/python/docker_entrypoint.sh @@ -5,7 +5,6 @@ env > .env if [ "$API" = true ]; then python3.12 api_endpoints.py & sleep 1 - python3.12 find_gateways.py & fi python3.12 main.py \ No newline at end of file diff --git a/python/requierements.txt b/python/requierements.txt index 17e3ae8..1a98dde 100644 --- a/python/requierements.txt +++ b/python/requierements.txt @@ -4,4 +4,5 @@ bs4 requests flask flask_cors -FindMyIP \ No newline at end of file +FindMyIP +paho-mqtt \ No newline at end of file diff --git a/python/src/api_endpoints.py b/python/src/api_endpoints.py index 0584760..79d830d 100644 --- a/python/src/api_endpoints.py +++ b/python/src/api_endpoints.py @@ -4,6 +4,7 @@ from flask import jsonify from flask import send_from_directory from flask_cors import CORS from flask_cors import cross_origin +from check_update import check_for_update class API: @@ -17,43 +18,44 @@ class API: self.app.config['CORS_HEADERS'] = 'Content-Type' # --------Static Routes------- - @self.app.route('/') + @self.app.route('/api') @cross_origin() def serve_root(): + workdir, filename = os.path.split(os.path.abspath(__file__)) + update = check_for_update() root_dict = { - "version": "24-07-03", - "mode": 1, + "version": { + "version": os.getenv('VERSION'), + "update_available": update.update_available if update is not None else None, + "up_to_date": update.up_to_date if update is not None else None, + "develop_version": update.development if update is not None else None, + }, + "mode": os.getenv('MODE'), + "name": os.getenv('NAME'), "info": { - "files_size_sum": self.get_file_size() + "files_size_sum": self.get_file_size(), + "files": os.listdir(f'{workdir}/data') } } return jsonify(root_dict) + @self.app.route('/api/json/') + @cross_origin() + def serve_json(path): + return send_from_directory(f'{workdir}/data', path) + @self.app.route('/charts') @cross_origin() def serve_index(): return send_from_directory('/src', 'chart.html') - @self.app.route('/json') - @cross_origin() - def serve_get_list_of_json(): - workdir, filename = os.path.split(os.path.abspath(__file__)) - return jsonify(os.listdir(f'{workdir}/data')) - - @self.app.route('/json/') - @cross_origin() - def serve_json(path): - - return send_from_directory(f'{workdir}/data', path) - + # --------Helpers------- def get_file_size(self): workdir, filename = os.path.split(os.path.abspath(__file__)) files = os.listdir(f'{workdir}/data') - sizes = 0.0 + sizes = 0 for file in files: - if file.endswith('.json'): - sizes += round(int(os.path.getsize(f'{workdir}/data/{file}')), 2) - + if file.endswith('.json'): sizes += os.path.getsize(f'{workdir}/data/{file}') return sizes diff --git a/python/src/check_update.py b/python/src/check_update.py new file mode 100644 index 0000000..63759f3 --- /dev/null +++ b/python/src/check_update.py @@ -0,0 +1,74 @@ +import os +import json +import requests + +DEBUG = True if os.environ.get('DEBUG') is not None else False + + +class State: + def __init__(self, version_int:int): + self.version_int:int = version_int + self.version_str:str = f"{str(version_int)[0]}{str(version_int)[1]}-{str(version_int)[2]}{str(version_int)[3]}-{str(version_int)[4]}{str(version_int)[5]}" + self.update_available:bool = False + self.up_to_date:bool = False + self.development:bool = False + + +class Release: + def __init__(self, data:dict): + self.name = data['name'] + self.tag_name = data['tag_name'] + self.description = data['description'] + self.created_at = data['created_at'] + self.released_at = data['released_at'] + self.upcoming_release = data['upcoming_release'] + self.version_int = int(self.tag_name.replace("-", "")) + + +def check_for_update(): + try: version_current = int(os.getenv('VERSION').replace("-", "")) + except: + print("Error getting current version") + return + project_id = 58341398 + request = f"https://gitlab.com/api/v4/projects/{project_id}/releases" + response = requests.get(url=request, timeout=1) + if not response.ok: return + + releases_json = json.loads(response.text) + # Date, Object + latest = [0, None] + for release_json in releases_json: + release = Release(release_json) + if release.version_int > latest[0]: + latest[0] = release.version_int + latest[1] = release + print(repr(latest)) if DEBUG else {} + + release = latest[1] + if release.version_int > version_current: + state = State(release.version_int) + state.update_available = True + return state + + elif release.version_int == version_current: + state = State(release.version_int) + state.up_to_date = True + return state + + else: + state = State(release.version_int) + state.development = True + return state + + +def print_state(state:State): + if state is None: return + print(f"Current version: {os.getenv('VERSION')}") + if state.update_available: + print(f"Update available: {state.version_str}") + if state.up_to_date: + print(f"Up to date") + if state.development: + print(f"Development Version") + print("") diff --git a/python/src/find_gateways.py b/python/src/find_gateways.py index e89ec39..2271f06 100644 --- a/python/src/find_gateways.py +++ b/python/src/find_gateways.py @@ -4,8 +4,11 @@ import FindMyIP import ipaddress import threading import requests +import os +DEBUG = True if os.environ.get('DEBUG') is not None else False max_threads = 50 +port = 8000 final = {} @@ -15,41 +18,37 @@ def check_port(ip, port): # sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # UDP socket.setdefaulttimeout(2.0) # seconds (float) result = sock.connect_ex((ip, port)) - if result == 0: - # print("Port is open") - final[ip] = "OPEN" - else: - # print("Port is closed/filtered") - final[ip] = "CLOSED" + if result == 0: final[ip] = True + else: final[ip] = False sock.close() + except: - final[ip] = "EXCEPTION" + pass -port = 8000 -local_ip = FindMyIP.internal() -local_ip = local_ip.split('.')[:-1] -local_ip.append("0") -local_ip = '.'.join(local_ip) +def scan_for_gateways(): + local_ip = FindMyIP.internal() + local_ip = local_ip.split('.')[:-1] + local_ip.append("0") + local_ip = '.'.join(local_ip) + print(f"Scan on {local_ip}/24 for port {port}") if DEBUG else {} -print(f"Scan on {local_ip}/24 for port {port}") + for ip in ipaddress.IPv4Network(f'{local_ip}/24'): + threading.Thread(target=check_port, args=[str(ip), port]).start() + # limit the number of threads. + while threading.active_count() > max_threads: sleep(1) -for ip in ipaddress.IPv4Network(f'{local_ip}/24'): - threading.Thread(target=check_port, args=[str(ip), port]).start() - # sleep(0.1) + sorted_ips = dict(sorted(final.items(), key=lambda item: tuple(map(int, item[0].split('.'))))) - # limit the number of threads. - while threading.active_count() > max_threads: - sleep(1) - -sorted_ips = dict(sorted(final.items(), key=lambda item: tuple(map(int, item[0].split('.'))))) - -for ip, state in sorted_ips.items(): - if state == "OPEN": - try: - response = requests.get(f'http://{ip}:{port}/') - if response.status_code == 200: - print(ip, state) - print(response.text) - except:pass + gateways = [] + for ip, state in sorted_ips.items(): + if state: + try: + response = requests.get(f'http://{ip}:{port}/api') + # TODO: Check if the API is for real a gateway + if response.status_code == 200: + print(ip, response.json()) if DEBUG else {} + gateways.append([ip, response.json()]) + except:pass + return gateways diff --git a/python/src/main.py b/python/src/main.py index 2fd502a..4561b4d 100644 --- a/python/src/main.py +++ b/python/src/main.py @@ -2,6 +2,8 @@ import os from discovery import start_discovery from log_data import log_to_json from loop import start_loop +from check_update import check_for_update +from check_update import print_state INTERVAL = 40 TIMEOUT = 20 @@ -17,8 +19,13 @@ if DEBUG: print(f"timeout: {timeout}") print(f"DOCKER: {DOCKER}") print(f"DEBUG: {DEBUG}") + print(f"VERSION: {os.getenv('VERSION')}") print("") +update_state = check_for_update() +print_state(update_state) + + if DOCKER: print("Running in docker") diff --git a/python/src/mqtt.py b/python/src/mqtt.py index 4640904..2464945 100644 --- a/python/src/mqtt.py +++ b/python/src/mqtt.py @@ -1 +1,15 @@ -# TODO +import paho.mqtt.client as MQTT + +topic_root = "/atc_mithermometer_gateway" + +mqtt = MQTT.Client(mqtt.CallbackAPIVersion.VERSION2) +mqtt.connect("192.168.178.140", 1883, 60) + + +def publish_measurement(mac): + topic = f"{topic_root}/measurements/{mac}" + topic_temp = f"{topic}/temperature" + topic_humid = f"{topic}/humidity" + topic_battery = f"{topic}/battery" + + mqtt.publish(topic_temp, ) diff --git a/run_gateway.sh b/run_gateway.sh index 4e7f3ff..47f71e0 100644 --- a/run_gateway.sh +++ b/run_gateway.sh @@ -1,14 +1,16 @@ TAG="latest" CONTAINER="dasmoorhuhn/atc-mithermometer-gateway" CONTAINER_NAME="ATC_MiThermometer_Gateway" -VOLUME=data +VOLUME=$(pwd)/data BACKGROUND="" TIME_ZONE="" +NAME="" INTERACTIVE=false BUILD=false API=false DEBUG=false +MODE=1 LOOP="0" TIMEOUT="0" @@ -20,6 +22,8 @@ HELP="USAGE: sh run_docker.sh [OPTIONS] \n [ -i | --interactive ] Start the container in interactive mode. That means, you can read the console output in real time and abort via STRG+C \n [ -a | --api ] Start with the API \n [ -v | --volume ] Set the volume, where the data from the gateway will be stored. Use relative path like /home/user/gateway/data \n +[ -n | --name ] Set a custom name for this gateway \n +[ -m2 | --mesh-gateway ] Set the mode to mesh gateway. This gateway is meant to expand the bluetooth radius. Default mode is main gateway \n [ -tz | --timezone ] Set the timezone. Default is Europe/Berlin \n [ -to | --timeout ] Set the timeout for the bluetooth scan. default is 20s \n [ -h | --help ] Get this dialog \n @@ -73,6 +77,14 @@ docker_run() { COMMAND="$COMMAND --env API=$API" fi + if [ "$MODE" != 1 ]; then + COMMAND="$COMMAND --env MODE=$MODE" + fi + + if [ "$NAME" != "" ]; then + COMMAND="$COMMAND --env NAME=$NAME" + fi + if [ "$DEBUG" = true ]; then COMMAND="$COMMAND --env DEBUG=$DEBUG" COMMAND="$COMMAND $CONTAINER:$TAG" @@ -114,6 +126,15 @@ while [ "$1" != "" ]; do VOLUME=$1 shift ;; + -n | --name ) + shift + NAME=$1 + shift + ;; + -m2 | --mesh-gateway) + MODE=2 + shift + ;; -tz | --timezone ) shift TIME_ZONE=$1 From 7415bb811229c06f79611723ba85df5962565a20 Mon Sep 17 00:00:00 2001 From: DasMoorhuhn Date: Sun, 7 Jul 2024 11:53:06 +0200 Subject: [PATCH 3/8] handle case if device config doesnt exists --- .idea/misc.xml | 2 +- .../devices.example.yml => devices.example.yml | 0 python/src/log_data.py | 4 ++-- run_gateway.sh | 15 +++++++++++++++ 4 files changed, 18 insertions(+), 3 deletions(-) rename python/src/devices.example.yml => devices.example.yml (100%) diff --git a/.idea/misc.xml b/.idea/misc.xml index 12391f3..afcdc7f 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,5 +3,5 @@ - + \ No newline at end of file diff --git a/python/src/devices.example.yml b/devices.example.yml similarity index 100% rename from python/src/devices.example.yml rename to devices.example.yml diff --git a/python/src/log_data.py b/python/src/log_data.py index 179a296..a198d52 100644 --- a/python/src/log_data.py +++ b/python/src/log_data.py @@ -30,8 +30,8 @@ def log_to_json(devices): "battery_percent": data_obj.battery_percent, "battery_volt": data_obj.battery_volt, "rssi": dev.rssi, - "name": from_config.name, - "room": from_config.room + "name": from_config.name if from_config is not None else "Unknown", + "room": from_config.room if from_config is not None else "Unknown" } data.append(measurements) diff --git a/run_gateway.sh b/run_gateway.sh index 47f71e0..0699743 100644 --- a/run_gateway.sh +++ b/run_gateway.sh @@ -35,12 +35,26 @@ if [ "$?" != 0 ]; then exit 1 fi + +check_for_devices_config() { + if [ ! -f devices.yml ]; then + touch devices.yml + echo 'devices: + - mac: A4:C1:38:00:00:00 + name: "my_room" + room: "my_room"' >> devices.yml + fi +} + + docker_run() { sudo killall -9 bluetoothd > /dev/null 2>&1 echo Killing old container... docker stop $CONTAINER_NAME > /dev/null 2>&1 docker container rm $CONTAINER_NAME > /dev/null 2>&1 + check_for_devices_config + COMMAND="docker run $BACKGROUND" COMMAND="$COMMAND --cap-add=SYS_ADMIN" COMMAND="$COMMAND --cap-add=NET_ADMIN" @@ -49,6 +63,7 @@ docker_run() { COMMAND="$COMMAND --restart=on-failure" COMMAND="$COMMAND --volume=/var/run/dbus/:/var/run/dbus/" COMMAND="$COMMAND --volume=$VOLUME:/src/data" + COMMAND="$COMMAND --volume=$PWD/devices.yml:/src/devices.yml" if [ "$INTERACTIVE" = true ]; then COMMAND="$COMMAND --interactive" From f74ce76369eaadff9ceffb49f4e427805814c7ae Mon Sep 17 00:00:00 2001 From: DasMoorhuhn Date: Wed, 28 Aug 2024 12:02:47 +0000 Subject: [PATCH 4/8] Update file README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index fbd0902..ff7c4ee 100644 --- a/README.md +++ b/README.md @@ -110,4 +110,5 @@ Coming when I develop it... # Resources - https://pythonspeed.com/articles/alpine-docker-python this article is nuts :D -- https://docs.docker.com/build/building/multi-stage/ \ No newline at end of file +- https://docs.docker.com/build/building/multi-stage/ +- https://github.com/jholtmann/ip_discovery \ No newline at end of file From 9747010caf2d6d158df4ad70619fc91b5e3b8493 Mon Sep 17 00:00:00 2001 From: DasMoorhuhn Date: Wed, 28 Aug 2024 18:47:04 +0200 Subject: [PATCH 5/8] add broadcast device discovery --- .idea/misc.xml | 2 +- python/docker_entrypoint.sh | 3 +- python/src/{discovery.py => ble_discovery.py} | 0 python/src/find_gateways.py | 101 +++++++++++------- python/src/helpers.py | 16 +++ python/src/loop.py | 2 +- python/src/main.py | 2 +- python/src/start_discovery_server.py | 2 + 8 files changed, 84 insertions(+), 44 deletions(-) rename python/src/{discovery.py => ble_discovery.py} (100%) create mode 100644 python/src/helpers.py create mode 100644 python/src/start_discovery_server.py diff --git a/.idea/misc.xml b/.idea/misc.xml index afcdc7f..12391f3 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -3,5 +3,5 @@ - + \ No newline at end of file diff --git a/python/docker_entrypoint.sh b/python/docker_entrypoint.sh index aeb780b..bafdaea 100644 --- a/python/docker_entrypoint.sh +++ b/python/docker_entrypoint.sh @@ -7,4 +7,5 @@ if [ "$API" = true ]; then sleep 1 fi -python3.12 main.py \ No newline at end of file +python3.12 start_discovery_server.py & +python3.12 main.py diff --git a/python/src/discovery.py b/python/src/ble_discovery.py similarity index 100% rename from python/src/discovery.py rename to python/src/ble_discovery.py diff --git a/python/src/find_gateways.py b/python/src/find_gateways.py index 2271f06..969c194 100644 --- a/python/src/find_gateways.py +++ b/python/src/find_gateways.py @@ -1,54 +1,75 @@ -from time import sleep -import socket -import FindMyIP -import ipaddress -import threading -import requests +# https://github.com/jholtmann/ip_discovery + import os +import FindMyIP +from socket import * +from helpers import get_unix_time DEBUG = True if os.environ.get('DEBUG') is not None else False -max_threads = 50 -port = 8000 -final = {} +DISCOVERY_ACK = 'IP_DISCOVERY_ACK'.encode() # ACK for broadcast +DISCOVERY_RSP_GTW = 'IP_DISCOVERY_RSP_GTW'.encode() # RSP for gateway +DISCOVERY_RSP_MSH = 'IP_DISCOVERY_RSP_MSH'.encode() # RSP for mesh +DISCOVERY_TIMEOUT = 20 +SOCKET_TIMEOUT = 5 +PORT_SERVER = 9434 +PORT_CLIENT = 9435 -def check_port(ip, port): +def start_discovery_server(): + sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) + sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1) + sock.setsockopt(SOL_SOCKET, SO_BROADCAST, 1) + server_address = ('', PORT_SERVER) + local_ip = FindMyIP.internal() try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # TCP - # sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # UDP - socket.setdefaulttimeout(2.0) # seconds (float) - result = sock.connect_ex((ip, port)) - if result == 0: final[ip] = True - else: final[ip] = False + sock.bind(server_address) + print("Started discovery socket") + while True: + data, addr = sock.recvfrom(4096) + print(f"Received a packet from {addr}") + print(f"{addr[0]} | {local_ip}") + print(f"{data} | {DISCOVERY_ACK}") + + if data == DISCOVERY_ACK: + print("ACK accepted") + if str(addr[0]) == str(local_ip): continue + print("IP accepted") + sock.sendto(DISCOVERY_RSP_GTW, (addr[0], PORT_CLIENT)) + print(f"Send ACK to {addr}") + + except Exception as err: + print(err) sock.close() - except: - pass - -def scan_for_gateways(): +def start_discovery_client(): + print("Started discovery client") + sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) + sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1) + sock.setsockopt(SOL_SOCKET, SO_BROADCAST, 1) + sock.settimeout(SOCKET_TIMEOUT) + server_address = ('255.255.255.255', PORT_SERVER) local_ip = FindMyIP.internal() - local_ip = local_ip.split('.')[:-1] - local_ip.append("0") - local_ip = '.'.join(local_ip) - print(f"Scan on {local_ip}/24 for port {port}") if DEBUG else {} + start_time_stamp = get_unix_time() + delta = round(get_unix_time() - start_time_stamp, 2) + discovered_devices = [] - for ip in ipaddress.IPv4Network(f'{local_ip}/24'): - threading.Thread(target=check_port, args=[str(ip), port]).start() - # limit the number of threads. - while threading.active_count() > max_threads: sleep(1) + try: + sock.bind(('', PORT_CLIENT)) + while True: + delta = round(get_unix_time() - start_time_stamp, 2) + print(delta) + sock.sendto(DISCOVERY_ACK, server_address) + data, addr = sock.recvfrom(4096) + print('IP: ' + str(addr[0])) + if data == DISCOVERY_RSP_GTW or data == DISCOVERY_RSP_GTW and addr[0] not in discovered_devices: + print('IP: ' + str(addr[0])) + discovered_devices.append(addr[0]) + except Exception as err: + print(err) - sorted_ips = dict(sorted(final.items(), key=lambda item: tuple(map(int, item[0].split('.'))))) + finally: + sock.close() - gateways = [] - for ip, state in sorted_ips.items(): - if state: - try: - response = requests.get(f'http://{ip}:{port}/api') - # TODO: Check if the API is for real a gateway - if response.status_code == 200: - print(ip, response.json()) if DEBUG else {} - gateways.append([ip, response.json()]) - except:pass - return gateways +# start_discovery_client() diff --git a/python/src/helpers.py b/python/src/helpers.py new file mode 100644 index 0000000..e7aaafc --- /dev/null +++ b/python/src/helpers.py @@ -0,0 +1,16 @@ +import datetime +import time + + +def get_time_now(): + date_now = datetime.datetime.now() + return str(date_now).split(" ")[0] + + +def get_unix_time(): + return time.time() + + +def get_date(): + date_now = datetime.datetime.now() + return str(date_now).split(" ") diff --git a/python/src/loop.py b/python/src/loop.py index 561a624..d3625f5 100644 --- a/python/src/loop.py +++ b/python/src/loop.py @@ -1,6 +1,6 @@ from time import sleep from log_data import log_to_json -from discovery import start_discovery +from ble_discovery import start_discovery def start_loop(interval=40, timeout=20): diff --git a/python/src/main.py b/python/src/main.py index 4561b4d..0f7ecac 100644 --- a/python/src/main.py +++ b/python/src/main.py @@ -1,5 +1,5 @@ import os -from discovery import start_discovery +from ble_discovery import start_discovery from log_data import log_to_json from loop import start_loop from check_update import check_for_update diff --git a/python/src/start_discovery_server.py b/python/src/start_discovery_server.py new file mode 100644 index 0000000..2eb9c21 --- /dev/null +++ b/python/src/start_discovery_server.py @@ -0,0 +1,2 @@ +from find_gateways import start_discovery_server +start_discovery_server() From 4c6689007dc085539e221ec903c0cd0f39e4abc7 Mon Sep 17 00:00:00 2001 From: DasMoorhuhn Date: Wed, 28 Aug 2024 22:06:50 +0200 Subject: [PATCH 6/8] Replace prints with logger --- .env | 0 Dockerfile | 2 +- python/src/ble_discovery.py | 10 ++++++---- python/src/check_update.py | 17 +++++++++-------- python/src/find_gateways.py | 36 +++++++++++++++++++----------------- python/src/log_data.py | 8 +++++--- python/src/logger.py | 18 ++++++++++++++++++ python/src/loop.py | 4 +++- python/src/main.py | 25 +++++++++++++------------ 9 files changed, 74 insertions(+), 46 deletions(-) create mode 100644 .env create mode 100644 python/src/logger.py diff --git a/.env b/.env new file mode 100644 index 0000000..e69de29 diff --git a/Dockerfile b/Dockerfile index 7fb1781..c795367 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,7 +38,7 @@ FROM python:3.12-alpine3.20 WORKDIR /src COPY ./python/src/ . COPY ./python/docker_entrypoint.sh / -RUN mkdir data +RUN mkdir -p data/log VOLUME /src/data RUN apk add --no-cache sudo bluez tzdata diff --git a/python/src/ble_discovery.py b/python/src/ble_discovery.py index 7d8b17f..5cecc10 100644 --- a/python/src/ble_discovery.py +++ b/python/src/ble_discovery.py @@ -3,6 +3,8 @@ from bluepy.btle import Scanner from data_class import Data from devices import get_device +from logger import get_logger +logger = get_logger(__name__) # This is the list, where the responses will be stored from the `handleDiscovery` devices = [] @@ -38,9 +40,9 @@ class ScanDelegate(DefaultDelegate): device_from_config = get_device(dev) - try:print(f"Device: {dev.addr.upper()} ({dev.addrType}), RSSI: {dev.rssi}dB, Room: {device_from_config.room}") - except:print(f"Device: {dev.addr.upper()} ({dev.addrType}), RSSI: {dev.rssi}dB, Room: ?") - print(f'\tTemp: {data_obj.temperature}°C, Humid: {data_obj.humidity}%, Batt: {data_obj.battery_percent}%\n') + try:logger.info(f"Device: {dev.addr.upper()} ({dev.addrType}), RSSI: {dev.rssi}dB, Room: {device_from_config.room}") + except:logger.info(f"Device: {dev.addr.upper()} ({dev.addrType}), RSSI: {dev.rssi}dB, Room: ?") + logger.info(f'\tTemp: {data_obj.temperature}°C, Humid: {data_obj.humidity}%, Batt: {data_obj.battery_percent}%') return True @@ -52,7 +54,7 @@ def cleanup(): def start_discovery(timeout=20): cleanup() global devices - print(f'Start discovery with timout {timeout}s...') + logger.info(f'Start discovery with timout {timeout}s...') scanner = Scanner().withDelegate(ScanDelegate()) scanner.scan(timeout=timeout, passive=False) diff --git a/python/src/check_update.py b/python/src/check_update.py index 63759f3..9efb65a 100644 --- a/python/src/check_update.py +++ b/python/src/check_update.py @@ -1,8 +1,10 @@ import os import json import requests +from logger import get_logger -DEBUG = True if os.environ.get('DEBUG') is not None else False +DEBUG = os.getenv('DEBUG') == 'true' +logger = get_logger(__name__) class State: @@ -28,7 +30,7 @@ class Release: def check_for_update(): try: version_current = int(os.getenv('VERSION').replace("-", "")) except: - print("Error getting current version") + logger.error("Error getting current version") return project_id = 58341398 request = f"https://gitlab.com/api/v4/projects/{project_id}/releases" @@ -43,7 +45,7 @@ def check_for_update(): if release.version_int > latest[0]: latest[0] = release.version_int latest[1] = release - print(repr(latest)) if DEBUG else {} + logger.debug(repr(latest)) release = latest[1] if release.version_int > version_current: @@ -64,11 +66,10 @@ def check_for_update(): def print_state(state:State): if state is None: return - print(f"Current version: {os.getenv('VERSION')}") + logger.info(f"Current version: {os.getenv('VERSION')}") if state.update_available: - print(f"Update available: {state.version_str}") + logger.info(f"Update available: {state.version_str}") if state.up_to_date: - print(f"Up to date") + logger.info(f"Up to date") if state.development: - print(f"Development Version") - print("") + logger.info(f"Development Version") diff --git a/python/src/find_gateways.py b/python/src/find_gateways.py index 969c194..481bac9 100644 --- a/python/src/find_gateways.py +++ b/python/src/find_gateways.py @@ -4,18 +4,20 @@ import os import FindMyIP from socket import * from helpers import get_unix_time +from logger import get_logger DEBUG = True if os.environ.get('DEBUG') is not None else False DISCOVERY_ACK = 'IP_DISCOVERY_ACK'.encode() # ACK for broadcast DISCOVERY_RSP_GTW = 'IP_DISCOVERY_RSP_GTW'.encode() # RSP for gateway DISCOVERY_RSP_MSH = 'IP_DISCOVERY_RSP_MSH'.encode() # RSP for mesh -DISCOVERY_TIMEOUT = 20 -SOCKET_TIMEOUT = 5 +DISCOVERY_TIMEOUT = 1 +SOCKET_TIMEOUT = 0.2 PORT_SERVER = 9434 PORT_CLIENT = 9435 def start_discovery_server(): + logger = get_logger(__name__) sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1) sock.setsockopt(SOL_SOCKET, SO_BROADCAST, 1) @@ -23,22 +25,22 @@ def start_discovery_server(): local_ip = FindMyIP.internal() try: sock.bind(server_address) - print("Started discovery socket") + logger.info("Started discovery socket") while True: data, addr = sock.recvfrom(4096) - print(f"Received a packet from {addr}") - print(f"{addr[0]} | {local_ip}") - print(f"{data} | {DISCOVERY_ACK}") + logger.debug(f"Received a packet from {addr}") + logger.debug(f"{addr[0]} | {local_ip}") + logger.debug(f"{data} | {DISCOVERY_ACK}") if data == DISCOVERY_ACK: - print("ACK accepted") + logger.debug("ACK accepted") if str(addr[0]) == str(local_ip): continue - print("IP accepted") + logger.debug("IP accepted") sock.sendto(DISCOVERY_RSP_GTW, (addr[0], PORT_CLIENT)) - print(f"Send ACK to {addr}") + logger.debug(f"Send ACK to {addr}") except Exception as err: - print(err) + logger.error(err) sock.close() @@ -49,27 +51,27 @@ def start_discovery_client(): sock.setsockopt(SOL_SOCKET, SO_BROADCAST, 1) sock.settimeout(SOCKET_TIMEOUT) server_address = ('255.255.255.255', PORT_SERVER) - local_ip = FindMyIP.internal() start_time_stamp = get_unix_time() delta = round(get_unix_time() - start_time_stamp, 2) discovered_devices = [] try: sock.bind(('', PORT_CLIENT)) - while True: + while delta <= DISCOVERY_TIMEOUT: delta = round(get_unix_time() - start_time_stamp, 2) - print(delta) sock.sendto(DISCOVERY_ACK, server_address) data, addr = sock.recvfrom(4096) - print('IP: ' + str(addr[0])) - if data == DISCOVERY_RSP_GTW or data == DISCOVERY_RSP_GTW and addr[0] not in discovered_devices: + if data == DISCOVERY_RSP_GTW or data == DISCOVERY_RSP_MSH: + if str(addr[0]) in discovered_devices: continue print('IP: ' + str(addr[0])) - discovered_devices.append(addr[0]) + discovered_devices.append(str(addr[0])) except Exception as err: print(err) finally: sock.close() + return discovered_devices -# start_discovery_client() +# devices = start_discovery_client() +# print(f"Devices: {devices}") diff --git a/python/src/log_data.py b/python/src/log_data.py index a198d52..032471a 100644 --- a/python/src/log_data.py +++ b/python/src/log_data.py @@ -3,8 +3,10 @@ import sys import json from data_class import Data from devices import Device +from logger import get_logger +logger = get_logger(__name__) -DEBUG = True if os.getenv('DEBUG') == 'true' else False +DEBUG = os.getenv('DEBUG') == 'true' def log_to_json(devices): @@ -15,7 +17,7 @@ def log_to_json(devices): data_obj: Data from_config: Device file_name = f'{workdir}/data/{str(data_obj.mac).replace(":", "-")}.json' - print(file_name) if DEBUG else {} + logger.debug(f"Save to {file_name}") try: with open(file_name, 'r') as file: data = json.load(file) @@ -35,7 +37,7 @@ def log_to_json(devices): } data.append(measurements) - print(measurements) if DEBUG else {} + logger.debug(measurements) with open(file_name, 'w') as file: file.write(json.dumps(data, indent=2)) diff --git a/python/src/logger.py b/python/src/logger.py new file mode 100644 index 0000000..8dfc870 --- /dev/null +++ b/python/src/logger.py @@ -0,0 +1,18 @@ +import os +import logging + + +def get_logger(logger_name:str, log_file='gateway.log'): + logger_name = logger_name.replace('__', '') + DEBUG = os.getenv('DEBUG') == 'true' + logger = logging.getLogger(logger_name) + logger.setLevel(logging.DEBUG) if DEBUG else logger.setLevel(logging.INFO) + handler = logging.FileHandler(filename=f'data/{log_file}', encoding='utf-8', mode='a') + formatter = logging.Formatter('%(asctime)s|%(levelname)s|%(name)s|:%(message)s') + handler.setFormatter(formatter) + stream_handler = logging.StreamHandler() + stream_handler.setFormatter(formatter) + logger.addHandler(stream_handler) + logger.addHandler(handler) + logger.info(f"Logger {logger_name} init done") + return logger diff --git a/python/src/loop.py b/python/src/loop.py index d3625f5..b75c2bf 100644 --- a/python/src/loop.py +++ b/python/src/loop.py @@ -1,10 +1,12 @@ from time import sleep from log_data import log_to_json from ble_discovery import start_discovery +from logger import get_logger +logger = get_logger(__name__) def start_loop(interval=40, timeout=20): - print(f"Starting loop with interval {interval}s") + logger.info(f"Starting loop with interval {interval}s") while True: devices = start_discovery(timeout=timeout) log_to_json(devices) diff --git a/python/src/main.py b/python/src/main.py index 0f7ecac..7b0cdf9 100644 --- a/python/src/main.py +++ b/python/src/main.py @@ -1,4 +1,5 @@ import os +from logger import get_logger from ble_discovery import start_discovery from log_data import log_to_json from loop import start_loop @@ -7,27 +8,27 @@ from check_update import print_state INTERVAL = 40 TIMEOUT = 20 -DOCKER = True if os.getenv('DOCKER') == 'true' else False -DEBUG = True if os.getenv('DEBUG') == 'true' else False +DOCKER = os.getenv('DOCKER') == 'true' +DEBUG = os.getenv('DEBUG') == 'true' interval = os.getenv('LOOP') timeout = os.getenv('TIMEOUT') -if DEBUG: - print(f"INTERVAL: {INTERVAL}") - print(f"TIMEOUT: {TIMEOUT}") - print(f"interval: {interval}") - print(f"timeout: {timeout}") - print(f"DOCKER: {DOCKER}") - print(f"DEBUG: {DEBUG}") - print(f"VERSION: {os.getenv('VERSION')}") - print("") +logger = get_logger(__name__) + +logger.debug(f"INTERVAL: {INTERVAL}") +logger.debug(f"TIMEOUT: {TIMEOUT}") +logger.debug(f"interval: {interval}") +logger.debug(f"timeout: {timeout}") +logger.debug(f"DOCKER: {DOCKER}") +logger.debug(f"DEBUG: {DEBUG}") +logger.debug(f"VERSION: {os.getenv('VERSION')}") update_state = check_for_update() print_state(update_state) if DOCKER: - print("Running in docker") + logger.info('Running in Docker') try:INTERVAL = int(interval) except:pass From b461894696a1fc79ce5ab3d0a4930700fe45f366 Mon Sep 17 00:00:00 2001 From: DasMoorhuhn Date: Wed, 28 Aug 2024 23:48:27 +0200 Subject: [PATCH 7/8] Add env file support and pass through log level --- .env | 0 .gitignore | 3 +- example.env | 10 ++++++ python/src/logger.py | 7 ++-- python/src/main.py | 24 +++++++------- run_gateway.sh | 76 ++++++++++++++++++++++++++++++++++++-------- 6 files changed, 92 insertions(+), 28 deletions(-) delete mode 100644 .env create mode 100644 example.env diff --git a/.env b/.env deleted file mode 100644 index e69de29..0000000 diff --git a/.gitignore b/.gitignore index 5078187..7b4b1f4 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ history.* data/ *.json *.iso -*.cow \ No newline at end of file +*.cow +.env \ No newline at end of file diff --git a/example.env b/example.env new file mode 100644 index 0000000..37d421d --- /dev/null +++ b/example.env @@ -0,0 +1,10 @@ +BACKGROUND= +TIME_ZONE= +NAME= +INTERACTIVE=true +BUILD=true +API=true +DEBUG=INFO +MODE=1 +LOOP=20 +TIMEOUT=20 \ No newline at end of file diff --git a/python/src/logger.py b/python/src/logger.py index 8dfc870..cd17c5e 100644 --- a/python/src/logger.py +++ b/python/src/logger.py @@ -4,9 +4,12 @@ import logging def get_logger(logger_name:str, log_file='gateway.log'): logger_name = logger_name.replace('__', '') - DEBUG = os.getenv('DEBUG') == 'true' + debug_level = os.getenv('DEBUG').upper() + if debug_level not in ('CRITICAL', 'ERROR', 'WARNING', 'WARN', 'INFO', 'DEBUG', 'NOTSET', 'FATAL'): + print(f'Loglevel "{debug_level}" is not supported.') + exit(0) logger = logging.getLogger(logger_name) - logger.setLevel(logging.DEBUG) if DEBUG else logger.setLevel(logging.INFO) + logger.setLevel(logging.getLevelName(debug_level)) handler = logging.FileHandler(filename=f'data/{log_file}', encoding='utf-8', mode='a') formatter = logging.Formatter('%(asctime)s|%(levelname)s|%(name)s|:%(message)s') handler.setFormatter(formatter) diff --git a/python/src/main.py b/python/src/main.py index 7b0cdf9..b9936d6 100644 --- a/python/src/main.py +++ b/python/src/main.py @@ -26,18 +26,20 @@ logger.debug(f"VERSION: {os.getenv('VERSION')}") update_state = check_for_update() print_state(update_state) +try: + if DOCKER: + logger.info('Running in Docker') -if DOCKER: - logger.info('Running in Docker') + try:INTERVAL = int(interval) + except:pass - try:INTERVAL = int(interval) - except:pass + try:TIMEOUT = int(timeout) + except:pass - try:TIMEOUT = int(timeout) - except:pass + if interval is None: log_to_json(start_discovery(timeout=TIMEOUT)) + else:start_loop(INTERVAL, TIMEOUT) - if interval is None: log_to_json(start_discovery(timeout=TIMEOUT)) - else:start_loop(INTERVAL, TIMEOUT) - -else: - start_loop(interval=40) + else: + start_loop(interval=40) +except Exception as err: + logger.error(err) diff --git a/run_gateway.sh b/run_gateway.sh index 0699743..c39f609 100644 --- a/run_gateway.sh +++ b/run_gateway.sh @@ -3,18 +3,8 @@ CONTAINER="dasmoorhuhn/atc-mithermometer-gateway" CONTAINER_NAME="ATC_MiThermometer_Gateway" VOLUME=$(pwd)/data -BACKGROUND="" -TIME_ZONE="" -NAME="" -INTERACTIVE=false -BUILD=false -API=false -DEBUG=false -MODE=1 -LOOP="0" -TIMEOUT="0" - -HELP="USAGE: sh run_docker.sh [OPTIONS] \n +HELP="Using any command line argument except -d bypasses the .env file\n\n +USAGE: sh run_docker.sh [OPTIONS] \n [ -d ] Run in Backgrund \n [ -t | --tag ] Set a docker tag. Default: latest \n [ -b | --build ] Build the image before running the container \n @@ -55,6 +45,43 @@ docker_run() { check_for_devices_config + if [ "$SKIP_ENV" = true ]; then + echo "Skip env file" + ENV_EXISTS=false + + BACKGROUND="" + TIME_ZONE="" + NAME="" + INTERACTIVE=true + BUILD=true + API=true + DEBUG="INFO" + MODE="1" + LOOP="40" + TIMEOUT="20" + else + if [ -e .env ] + then + echo Loading .env file + export $(cat .env | xargs) + ENV_EXISTS=true + else + echo No env file found + ENV_EXISTS=false + + BACKGROUND="" + TIME_ZONE="" + NAME="" + INTERACTIVE=true + BUILD=true + API=true + DEBUG="INFO" + MODE="1" + LOOP="40" + TIMEOUT="20" + fi + fi + COMMAND="docker run $BACKGROUND" COMMAND="$COMMAND --cap-add=SYS_ADMIN" COMMAND="$COMMAND --cap-add=NET_ADMIN" @@ -65,6 +92,10 @@ docker_run() { COMMAND="$COMMAND --volume=$VOLUME:/src/data" COMMAND="$COMMAND --volume=$PWD/devices.yml:/src/devices.yml" + if [ "$ENV_EXISTS" = true ]; then + COMMAND="$COMMAND --env-file .env" + fi + if [ "$INTERACTIVE" = true ]; then COMMAND="$COMMAND --interactive" COMMAND="$COMMAND --tty" @@ -100,7 +131,7 @@ docker_run() { COMMAND="$COMMAND --env NAME=$NAME" fi - if [ "$DEBUG" = true ]; then + if [ "$DEBUG" = "DEBUG" ]; then COMMAND="$COMMAND --env DEBUG=$DEBUG" COMMAND="$COMMAND $CONTAINER:$TAG" echo @@ -108,6 +139,7 @@ docker_run() { echo echo DEBUG MODE else + COMMAND="$COMMAND --env DEBUG=$DEBUG" COMMAND="$COMMAND $CONTAINER:$TAG" fi @@ -120,53 +152,68 @@ docker_run() { while [ "$1" != "" ]; do case $1 in + -se | --skip-env-file ) + SKIP_ENV=true + shift + ;; -d ) BACKGROUND="-d" shift ;; --debug ) - DEBUG=true + shift + DEBUG=$1 + SKIP_ENV=true shift ;; -a | --api) API=true + SKIP_ENV=true shift ;; -b | --build ) BUILD=true + SKIP_ENV=true shift ;; -v | --volume ) shift VOLUME=$1 + SKIP_ENV=true shift ;; -n | --name ) shift NAME=$1 + SKIP_ENV=true shift ;; -m2 | --mesh-gateway) MODE=2 + SKIP_ENV=true shift ;; -tz | --timezone ) shift TIME_ZONE=$1 + SKIP_ENV=true shift ;; -to | --timeout ) shift TIMEOUT=$1 + SKIP_ENV=true shift ;; -t | --tag ) shift TAG=$1 + SKIP_ENV=true shift ;; -l | --loop ) shift + SKIP_ENV=true firstchar=`echo $1 | cut -c1-1` if [ "$firstchar" = "-" ]; then LOOP=0 @@ -179,6 +226,7 @@ while [ "$1" != "" ]; do ;; -i | --interactive ) INTERACTIVE=true + SKIP_ENV=true shift ;; -h | --help ) From 1a6e6bd1ba57634bc2875de89e813c07177e84cb Mon Sep 17 00:00:00 2001 From: DasMoorhuhn Date: Thu, 29 Aug 2024 00:14:55 +0200 Subject: [PATCH 8/8] removed unused debug statements --- python/src/check_update.py | 1 - python/src/log_data.py | 2 -- python/src/main.py | 2 +- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/python/src/check_update.py b/python/src/check_update.py index 9efb65a..8327453 100644 --- a/python/src/check_update.py +++ b/python/src/check_update.py @@ -3,7 +3,6 @@ import json import requests from logger import get_logger -DEBUG = os.getenv('DEBUG') == 'true' logger = get_logger(__name__) diff --git a/python/src/log_data.py b/python/src/log_data.py index 032471a..1c8b4c3 100644 --- a/python/src/log_data.py +++ b/python/src/log_data.py @@ -6,8 +6,6 @@ from devices import Device from logger import get_logger logger = get_logger(__name__) -DEBUG = os.getenv('DEBUG') == 'true' - def log_to_json(devices): workdir, filename = os.path.split(os.path.abspath(__file__)) diff --git a/python/src/main.py b/python/src/main.py index b9936d6..72a9395 100644 --- a/python/src/main.py +++ b/python/src/main.py @@ -9,7 +9,7 @@ from check_update import print_state INTERVAL = 40 TIMEOUT = 20 DOCKER = os.getenv('DOCKER') == 'true' -DEBUG = os.getenv('DEBUG') == 'true' +DEBUG = os.getenv('DEBUG') interval = os.getenv('LOOP') timeout = os.getenv('TIMEOUT')