diff --git a/.gitignore b/.gitignore index 5078187..7b4b1f4 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ history.* data/ *.json *.iso -*.cow \ No newline at end of file +*.cow +.env \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index bf958e5..c795367 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,13 +38,16 @@ FROM python:3.12-alpine3.20 WORKDIR /src COPY ./python/src/ . COPY ./python/docker_entrypoint.sh / -RUN mkdir data +RUN mkdir -p data/log VOLUME /src/data RUN apk add --no-cache sudo bluez tzdata ENV TZ=Europe/Berlin ENV DOCKER=true ENV API=false +ENV NAME=ATC_MiThermometer_Gateway +ENV VERSION=24-07-03 +ENV MODE=1 # Copy pips from the pip build stage COPY --from=pip_build_stage /usr/local/lib/python3.12/site-packages /usr/local/lib/python3.12/site-packages diff --git a/README.md b/README.md index fbd0902..ff7c4ee 100644 --- a/README.md +++ b/README.md @@ -110,4 +110,5 @@ Coming when I develop it... # Resources - https://pythonspeed.com/articles/alpine-docker-python this article is nuts :D -- https://docs.docker.com/build/building/multi-stage/ \ No newline at end of file +- https://docs.docker.com/build/building/multi-stage/ +- https://github.com/jholtmann/ip_discovery \ No newline at end of file diff --git a/python/src/devices.example.yml b/devices.example.yml similarity index 100% rename from python/src/devices.example.yml rename to devices.example.yml diff --git a/example.env b/example.env new file mode 100644 index 0000000..37d421d --- /dev/null +++ b/example.env @@ -0,0 +1,10 @@ +BACKGROUND= +TIME_ZONE= +NAME= +INTERACTIVE=true +BUILD=true +API=true +DEBUG=INFO +MODE=1 +LOOP=20 +TIMEOUT=20 \ No newline at end of file diff --git a/python/docker_entrypoint.sh b/python/docker_entrypoint.sh index aeb780b..bafdaea 100644 --- a/python/docker_entrypoint.sh +++ b/python/docker_entrypoint.sh @@ -7,4 +7,5 @@ if [ "$API" = true ]; then sleep 1 fi -python3.12 main.py \ No newline at end of file +python3.12 start_discovery_server.py & +python3.12 main.py diff --git a/python/requierements.txt b/python/requierements.txt index b7ed443..1a98dde 100644 --- a/python/requierements.txt +++ b/python/requierements.txt @@ -3,4 +3,6 @@ pyyaml bs4 requests flask -flask_cors \ No newline at end of file +flask_cors +FindMyIP +paho-mqtt \ No newline at end of file diff --git a/python/src/api_endpoints.py b/python/src/api_endpoints.py index 6448f3b..79d830d 100644 --- a/python/src/api_endpoints.py +++ b/python/src/api_endpoints.py @@ -4,6 +4,7 @@ from flask import jsonify from flask import send_from_directory from flask_cors import CORS from flask_cors import cross_origin +from check_update import check_for_update class API: @@ -17,22 +18,45 @@ class API: self.app.config['CORS_HEADERS'] = 'Content-Type' # --------Static Routes------- + @self.app.route('/api') + @cross_origin() + def serve_root(): + workdir, filename = os.path.split(os.path.abspath(__file__)) + update = check_for_update() + root_dict = { + "version": { + "version": os.getenv('VERSION'), + "update_available": update.update_available if update is not None else None, + "up_to_date": update.up_to_date if update is not None else None, + "develop_version": update.development if update is not None else None, + }, + "mode": os.getenv('MODE'), + "name": os.getenv('NAME'), + "info": { + "files_size_sum": self.get_file_size(), + "files": os.listdir(f'{workdir}/data') + } + } + return jsonify(root_dict) + + @self.app.route('/api/json/') + @cross_origin() + def serve_json(path): + return send_from_directory(f'{workdir}/data', path) + @self.app.route('/charts') @cross_origin() def serve_index(): return send_from_directory('/src', 'chart.html') - @self.app.route('/json') - @cross_origin() - def serve_get_list_of_json(): - workdir, filename = os.path.split(os.path.abspath(__file__)) - return jsonify(os.listdir(f'{workdir}/data')) - - @self.app.route('/json/') - @cross_origin() - def serve_json(path): - workdir, filename = os.path.split(os.path.abspath(__file__)) - return send_from_directory(f'{workdir}/data', path) + # --------Helpers------- + def get_file_size(self): + workdir, filename = os.path.split(os.path.abspath(__file__)) + files = os.listdir(f'{workdir}/data') + sizes = 0 + for file in files: + if file.endswith('.json'): sizes += os.path.getsize(f'{workdir}/data/{file}') + return sizes api = API() diff --git a/python/src/discovery.py b/python/src/ble_discovery.py similarity index 74% rename from python/src/discovery.py rename to python/src/ble_discovery.py index 7d8b17f..5cecc10 100644 --- a/python/src/discovery.py +++ b/python/src/ble_discovery.py @@ -3,6 +3,8 @@ from bluepy.btle import Scanner from data_class import Data from devices import get_device +from logger import get_logger +logger = get_logger(__name__) # This is the list, where the responses will be stored from the `handleDiscovery` devices = [] @@ -38,9 +40,9 @@ class ScanDelegate(DefaultDelegate): device_from_config = get_device(dev) - try:print(f"Device: {dev.addr.upper()} ({dev.addrType}), RSSI: {dev.rssi}dB, Room: {device_from_config.room}") - except:print(f"Device: {dev.addr.upper()} ({dev.addrType}), RSSI: {dev.rssi}dB, Room: ?") - print(f'\tTemp: {data_obj.temperature}°C, Humid: {data_obj.humidity}%, Batt: {data_obj.battery_percent}%\n') + try:logger.info(f"Device: {dev.addr.upper()} ({dev.addrType}), RSSI: {dev.rssi}dB, Room: {device_from_config.room}") + except:logger.info(f"Device: {dev.addr.upper()} ({dev.addrType}), RSSI: {dev.rssi}dB, Room: ?") + logger.info(f'\tTemp: {data_obj.temperature}°C, Humid: {data_obj.humidity}%, Batt: {data_obj.battery_percent}%') return True @@ -52,7 +54,7 @@ def cleanup(): def start_discovery(timeout=20): cleanup() global devices - print(f'Start discovery with timout {timeout}s...') + logger.info(f'Start discovery with timout {timeout}s...') scanner = Scanner().withDelegate(ScanDelegate()) scanner.scan(timeout=timeout, passive=False) diff --git a/python/src/check_update.py b/python/src/check_update.py new file mode 100644 index 0000000..8327453 --- /dev/null +++ b/python/src/check_update.py @@ -0,0 +1,74 @@ +import os +import json +import requests +from logger import get_logger + +logger = get_logger(__name__) + + +class State: + def __init__(self, version_int:int): + self.version_int:int = version_int + self.version_str:str = f"{str(version_int)[0]}{str(version_int)[1]}-{str(version_int)[2]}{str(version_int)[3]}-{str(version_int)[4]}{str(version_int)[5]}" + self.update_available:bool = False + self.up_to_date:bool = False + self.development:bool = False + + +class Release: + def __init__(self, data:dict): + self.name = data['name'] + self.tag_name = data['tag_name'] + self.description = data['description'] + self.created_at = data['created_at'] + self.released_at = data['released_at'] + self.upcoming_release = data['upcoming_release'] + self.version_int = int(self.tag_name.replace("-", "")) + + +def check_for_update(): + try: version_current = int(os.getenv('VERSION').replace("-", "")) + except: + logger.error("Error getting current version") + return + project_id = 58341398 + request = f"https://gitlab.com/api/v4/projects/{project_id}/releases" + response = requests.get(url=request, timeout=1) + if not response.ok: return + + releases_json = json.loads(response.text) + # Date, Object + latest = [0, None] + for release_json in releases_json: + release = Release(release_json) + if release.version_int > latest[0]: + latest[0] = release.version_int + latest[1] = release + logger.debug(repr(latest)) + + release = latest[1] + if release.version_int > version_current: + state = State(release.version_int) + state.update_available = True + return state + + elif release.version_int == version_current: + state = State(release.version_int) + state.up_to_date = True + return state + + else: + state = State(release.version_int) + state.development = True + return state + + +def print_state(state:State): + if state is None: return + logger.info(f"Current version: {os.getenv('VERSION')}") + if state.update_available: + logger.info(f"Update available: {state.version_str}") + if state.up_to_date: + logger.info(f"Up to date") + if state.development: + logger.info(f"Development Version") diff --git a/python/src/find_gateways.py b/python/src/find_gateways.py new file mode 100644 index 0000000..481bac9 --- /dev/null +++ b/python/src/find_gateways.py @@ -0,0 +1,77 @@ +# https://github.com/jholtmann/ip_discovery + +import os +import FindMyIP +from socket import * +from helpers import get_unix_time +from logger import get_logger + +DEBUG = True if os.environ.get('DEBUG') is not None else False +DISCOVERY_ACK = 'IP_DISCOVERY_ACK'.encode() # ACK for broadcast +DISCOVERY_RSP_GTW = 'IP_DISCOVERY_RSP_GTW'.encode() # RSP for gateway +DISCOVERY_RSP_MSH = 'IP_DISCOVERY_RSP_MSH'.encode() # RSP for mesh +DISCOVERY_TIMEOUT = 1 +SOCKET_TIMEOUT = 0.2 +PORT_SERVER = 9434 +PORT_CLIENT = 9435 + + +def start_discovery_server(): + logger = get_logger(__name__) + sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) + sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1) + sock.setsockopt(SOL_SOCKET, SO_BROADCAST, 1) + server_address = ('', PORT_SERVER) + local_ip = FindMyIP.internal() + try: + sock.bind(server_address) + logger.info("Started discovery socket") + while True: + data, addr = sock.recvfrom(4096) + logger.debug(f"Received a packet from {addr}") + logger.debug(f"{addr[0]} | {local_ip}") + logger.debug(f"{data} | {DISCOVERY_ACK}") + + if data == DISCOVERY_ACK: + logger.debug("ACK accepted") + if str(addr[0]) == str(local_ip): continue + logger.debug("IP accepted") + sock.sendto(DISCOVERY_RSP_GTW, (addr[0], PORT_CLIENT)) + logger.debug(f"Send ACK to {addr}") + + except Exception as err: + logger.error(err) + sock.close() + + +def start_discovery_client(): + print("Started discovery client") + sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP) + sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1) + sock.setsockopt(SOL_SOCKET, SO_BROADCAST, 1) + sock.settimeout(SOCKET_TIMEOUT) + server_address = ('255.255.255.255', PORT_SERVER) + start_time_stamp = get_unix_time() + delta = round(get_unix_time() - start_time_stamp, 2) + discovered_devices = [] + + try: + sock.bind(('', PORT_CLIENT)) + while delta <= DISCOVERY_TIMEOUT: + delta = round(get_unix_time() - start_time_stamp, 2) + sock.sendto(DISCOVERY_ACK, server_address) + data, addr = sock.recvfrom(4096) + if data == DISCOVERY_RSP_GTW or data == DISCOVERY_RSP_MSH: + if str(addr[0]) in discovered_devices: continue + print('IP: ' + str(addr[0])) + discovered_devices.append(str(addr[0])) + except Exception as err: + print(err) + + finally: + sock.close() + return discovered_devices + + +# devices = start_discovery_client() +# print(f"Devices: {devices}") diff --git a/python/src/helpers.py b/python/src/helpers.py new file mode 100644 index 0000000..e7aaafc --- /dev/null +++ b/python/src/helpers.py @@ -0,0 +1,16 @@ +import datetime +import time + + +def get_time_now(): + date_now = datetime.datetime.now() + return str(date_now).split(" ")[0] + + +def get_unix_time(): + return time.time() + + +def get_date(): + date_now = datetime.datetime.now() + return str(date_now).split(" ") diff --git a/python/src/log_data.py b/python/src/log_data.py index 179a296..1c8b4c3 100644 --- a/python/src/log_data.py +++ b/python/src/log_data.py @@ -3,8 +3,8 @@ import sys import json from data_class import Data from devices import Device - -DEBUG = True if os.getenv('DEBUG') == 'true' else False +from logger import get_logger +logger = get_logger(__name__) def log_to_json(devices): @@ -15,7 +15,7 @@ def log_to_json(devices): data_obj: Data from_config: Device file_name = f'{workdir}/data/{str(data_obj.mac).replace(":", "-")}.json' - print(file_name) if DEBUG else {} + logger.debug(f"Save to {file_name}") try: with open(file_name, 'r') as file: data = json.load(file) @@ -30,12 +30,12 @@ def log_to_json(devices): "battery_percent": data_obj.battery_percent, "battery_volt": data_obj.battery_volt, "rssi": dev.rssi, - "name": from_config.name, - "room": from_config.room + "name": from_config.name if from_config is not None else "Unknown", + "room": from_config.room if from_config is not None else "Unknown" } data.append(measurements) - print(measurements) if DEBUG else {} + logger.debug(measurements) with open(file_name, 'w') as file: file.write(json.dumps(data, indent=2)) diff --git a/python/src/logger.py b/python/src/logger.py new file mode 100644 index 0000000..cd17c5e --- /dev/null +++ b/python/src/logger.py @@ -0,0 +1,21 @@ +import os +import logging + + +def get_logger(logger_name:str, log_file='gateway.log'): + logger_name = logger_name.replace('__', '') + debug_level = os.getenv('DEBUG').upper() + if debug_level not in ('CRITICAL', 'ERROR', 'WARNING', 'WARN', 'INFO', 'DEBUG', 'NOTSET', 'FATAL'): + print(f'Loglevel "{debug_level}" is not supported.') + exit(0) + logger = logging.getLogger(logger_name) + logger.setLevel(logging.getLevelName(debug_level)) + handler = logging.FileHandler(filename=f'data/{log_file}', encoding='utf-8', mode='a') + formatter = logging.Formatter('%(asctime)s|%(levelname)s|%(name)s|:%(message)s') + handler.setFormatter(formatter) + stream_handler = logging.StreamHandler() + stream_handler.setFormatter(formatter) + logger.addHandler(stream_handler) + logger.addHandler(handler) + logger.info(f"Logger {logger_name} init done") + return logger diff --git a/python/src/loop.py b/python/src/loop.py index 561a624..b75c2bf 100644 --- a/python/src/loop.py +++ b/python/src/loop.py @@ -1,10 +1,12 @@ from time import sleep from log_data import log_to_json -from discovery import start_discovery +from ble_discovery import start_discovery +from logger import get_logger +logger = get_logger(__name__) def start_loop(interval=40, timeout=20): - print(f"Starting loop with interval {interval}s") + logger.info(f"Starting loop with interval {interval}s") while True: devices = start_discovery(timeout=timeout) log_to_json(devices) diff --git a/python/src/main.py b/python/src/main.py index 2fd502a..72a9395 100644 --- a/python/src/main.py +++ b/python/src/main.py @@ -1,35 +1,45 @@ import os -from discovery import start_discovery +from logger import get_logger +from ble_discovery import start_discovery from log_data import log_to_json from loop import start_loop +from check_update import check_for_update +from check_update import print_state INTERVAL = 40 TIMEOUT = 20 -DOCKER = True if os.getenv('DOCKER') == 'true' else False -DEBUG = True if os.getenv('DEBUG') == 'true' else False +DOCKER = os.getenv('DOCKER') == 'true' +DEBUG = os.getenv('DEBUG') interval = os.getenv('LOOP') timeout = os.getenv('TIMEOUT') -if DEBUG: - print(f"INTERVAL: {INTERVAL}") - print(f"TIMEOUT: {TIMEOUT}") - print(f"interval: {interval}") - print(f"timeout: {timeout}") - print(f"DOCKER: {DOCKER}") - print(f"DEBUG: {DEBUG}") - print("") +logger = get_logger(__name__) -if DOCKER: - print("Running in docker") +logger.debug(f"INTERVAL: {INTERVAL}") +logger.debug(f"TIMEOUT: {TIMEOUT}") +logger.debug(f"interval: {interval}") +logger.debug(f"timeout: {timeout}") +logger.debug(f"DOCKER: {DOCKER}") +logger.debug(f"DEBUG: {DEBUG}") +logger.debug(f"VERSION: {os.getenv('VERSION')}") - try:INTERVAL = int(interval) - except:pass +update_state = check_for_update() +print_state(update_state) - try:TIMEOUT = int(timeout) - except:pass +try: + if DOCKER: + logger.info('Running in Docker') - if interval is None: log_to_json(start_discovery(timeout=TIMEOUT)) - else:start_loop(INTERVAL, TIMEOUT) + try:INTERVAL = int(interval) + except:pass -else: - start_loop(interval=40) + try:TIMEOUT = int(timeout) + except:pass + + if interval is None: log_to_json(start_discovery(timeout=TIMEOUT)) + else:start_loop(INTERVAL, TIMEOUT) + + else: + start_loop(interval=40) +except Exception as err: + logger.error(err) diff --git a/python/src/mqtt.py b/python/src/mqtt.py index 4640904..2464945 100644 --- a/python/src/mqtt.py +++ b/python/src/mqtt.py @@ -1 +1,15 @@ -# TODO +import paho.mqtt.client as MQTT + +topic_root = "/atc_mithermometer_gateway" + +mqtt = MQTT.Client(mqtt.CallbackAPIVersion.VERSION2) +mqtt.connect("192.168.178.140", 1883, 60) + + +def publish_measurement(mac): + topic = f"{topic_root}/measurements/{mac}" + topic_temp = f"{topic}/temperature" + topic_humid = f"{topic}/humidity" + topic_battery = f"{topic}/battery" + + mqtt.publish(topic_temp, ) diff --git a/python/src/start_discovery_server.py b/python/src/start_discovery_server.py new file mode 100644 index 0000000..2eb9c21 --- /dev/null +++ b/python/src/start_discovery_server.py @@ -0,0 +1,2 @@ +from find_gateways import start_discovery_server +start_discovery_server() diff --git a/run_gateway.sh b/run_gateway.sh index 4e7f3ff..c39f609 100644 --- a/run_gateway.sh +++ b/run_gateway.sh @@ -1,18 +1,10 @@ TAG="latest" CONTAINER="dasmoorhuhn/atc-mithermometer-gateway" CONTAINER_NAME="ATC_MiThermometer_Gateway" -VOLUME=data +VOLUME=$(pwd)/data -BACKGROUND="" -TIME_ZONE="" -INTERACTIVE=false -BUILD=false -API=false -DEBUG=false -LOOP="0" -TIMEOUT="0" - -HELP="USAGE: sh run_docker.sh [OPTIONS] \n +HELP="Using any command line argument except -d bypasses the .env file\n\n +USAGE: sh run_docker.sh [OPTIONS] \n [ -d ] Run in Backgrund \n [ -t | --tag ] Set a docker tag. Default: latest \n [ -b | --build ] Build the image before running the container \n @@ -20,6 +12,8 @@ HELP="USAGE: sh run_docker.sh [OPTIONS] \n [ -i | --interactive ] Start the container in interactive mode. That means, you can read the console output in real time and abort via STRG+C \n [ -a | --api ] Start with the API \n [ -v | --volume ] Set the volume, where the data from the gateway will be stored. Use relative path like /home/user/gateway/data \n +[ -n | --name ] Set a custom name for this gateway \n +[ -m2 | --mesh-gateway ] Set the mode to mesh gateway. This gateway is meant to expand the bluetooth radius. Default mode is main gateway \n [ -tz | --timezone ] Set the timezone. Default is Europe/Berlin \n [ -to | --timeout ] Set the timeout for the bluetooth scan. default is 20s \n [ -h | --help ] Get this dialog \n @@ -31,12 +25,63 @@ if [ "$?" != 0 ]; then exit 1 fi + +check_for_devices_config() { + if [ ! -f devices.yml ]; then + touch devices.yml + echo 'devices: + - mac: A4:C1:38:00:00:00 + name: "my_room" + room: "my_room"' >> devices.yml + fi +} + + docker_run() { sudo killall -9 bluetoothd > /dev/null 2>&1 echo Killing old container... docker stop $CONTAINER_NAME > /dev/null 2>&1 docker container rm $CONTAINER_NAME > /dev/null 2>&1 + check_for_devices_config + + if [ "$SKIP_ENV" = true ]; then + echo "Skip env file" + ENV_EXISTS=false + + BACKGROUND="" + TIME_ZONE="" + NAME="" + INTERACTIVE=true + BUILD=true + API=true + DEBUG="INFO" + MODE="1" + LOOP="40" + TIMEOUT="20" + else + if [ -e .env ] + then + echo Loading .env file + export $(cat .env | xargs) + ENV_EXISTS=true + else + echo No env file found + ENV_EXISTS=false + + BACKGROUND="" + TIME_ZONE="" + NAME="" + INTERACTIVE=true + BUILD=true + API=true + DEBUG="INFO" + MODE="1" + LOOP="40" + TIMEOUT="20" + fi + fi + COMMAND="docker run $BACKGROUND" COMMAND="$COMMAND --cap-add=SYS_ADMIN" COMMAND="$COMMAND --cap-add=NET_ADMIN" @@ -45,6 +90,11 @@ docker_run() { COMMAND="$COMMAND --restart=on-failure" COMMAND="$COMMAND --volume=/var/run/dbus/:/var/run/dbus/" COMMAND="$COMMAND --volume=$VOLUME:/src/data" + COMMAND="$COMMAND --volume=$PWD/devices.yml:/src/devices.yml" + + if [ "$ENV_EXISTS" = true ]; then + COMMAND="$COMMAND --env-file .env" + fi if [ "$INTERACTIVE" = true ]; then COMMAND="$COMMAND --interactive" @@ -73,7 +123,15 @@ docker_run() { COMMAND="$COMMAND --env API=$API" fi - if [ "$DEBUG" = true ]; then + if [ "$MODE" != 1 ]; then + COMMAND="$COMMAND --env MODE=$MODE" + fi + + if [ "$NAME" != "" ]; then + COMMAND="$COMMAND --env NAME=$NAME" + fi + + if [ "$DEBUG" = "DEBUG" ]; then COMMAND="$COMMAND --env DEBUG=$DEBUG" COMMAND="$COMMAND $CONTAINER:$TAG" echo @@ -81,6 +139,7 @@ docker_run() { echo echo DEBUG MODE else + COMMAND="$COMMAND --env DEBUG=$DEBUG" COMMAND="$COMMAND $CONTAINER:$TAG" fi @@ -93,44 +152,68 @@ docker_run() { while [ "$1" != "" ]; do case $1 in + -se | --skip-env-file ) + SKIP_ENV=true + shift + ;; -d ) BACKGROUND="-d" shift ;; --debug ) - DEBUG=true + shift + DEBUG=$1 + SKIP_ENV=true shift ;; -a | --api) API=true + SKIP_ENV=true shift ;; -b | --build ) BUILD=true + SKIP_ENV=true shift ;; -v | --volume ) shift VOLUME=$1 + SKIP_ENV=true + shift + ;; + -n | --name ) + shift + NAME=$1 + SKIP_ENV=true + shift + ;; + -m2 | --mesh-gateway) + MODE=2 + SKIP_ENV=true shift ;; -tz | --timezone ) shift TIME_ZONE=$1 + SKIP_ENV=true shift ;; -to | --timeout ) shift TIMEOUT=$1 + SKIP_ENV=true shift ;; -t | --tag ) shift TAG=$1 + SKIP_ENV=true shift ;; -l | --loop ) shift + SKIP_ENV=true firstchar=`echo $1 | cut -c1-1` if [ "$firstchar" = "-" ]; then LOOP=0 @@ -143,6 +226,7 @@ while [ "$1" != "" ]; do ;; -i | --interactive ) INTERACTIVE=true + SKIP_ENV=true shift ;; -h | --help )