Merge branch 'develop' into 'main'

Develop

See merge request DasMoorhuhn/atc_mithermometer_gateway!3

.gitignore

@@ -5,3 +5,4 @@ data/
 *.json
 *.iso
 *.cow
+.env

Dockerfile

@@ -38,13 +38,16 @@ FROM python:3.12-alpine3.20
 WORKDIR /src
 COPY ./python/src/ .
 COPY ./python/docker_entrypoint.sh /
-RUN mkdir data
+RUN mkdir -p data/log
 VOLUME /src/data
 
 RUN apk add --no-cache sudo bluez tzdata
 ENV TZ=Europe/Berlin
 ENV DOCKER=true
 ENV API=false
+ENV NAME=ATC_MiThermometer_Gateway
+ENV VERSION=24-07-03
+ENV MODE=1
 
 # Copy pips from the pip build stage
 COPY --from=pip_build_stage /usr/local/lib/python3.12/site-packages /usr/local/lib/python3.12/site-packages

README.md

@@ -111,3 +111,4 @@ Coming when I develop it...
 # Resources
 - https://pythonspeed.com/articles/alpine-docker-python this article is nuts :D
 - https://docs.docker.com/build/building/multi-stage/
+- https://github.com/jholtmann/ip_discovery

example.env

@@ -0,0 +1,10 @@
+BACKGROUND=
+TIME_ZONE=
+NAME=
+INTERACTIVE=true
+BUILD=true
+API=true
+DEBUG=INFO
+MODE=1
+LOOP=20
+TIMEOUT=20

python/docker_entrypoint.sh

@@ -7,4 +7,5 @@ if [ "$API" = true ]; then
   sleep 1
 fi
 
+python3.12 start_discovery_server.py &
 python3.12 main.py

python/requierements.txt

@@ -4,3 +4,5 @@ bs4
 requests
 flask
 flask_cors
+FindMyIP
+paho-mqtt

python/src/api_endpoints.py

@@ -4,6 +4,7 @@ from flask import jsonify
 from flask import send_from_directory
 from flask_cors import CORS
 from flask_cors import cross_origin
+from check_update import check_for_update
 
 
 class API:
@@ -17,22 +18,45 @@ class API:
     self.app.config['CORS_HEADERS'] = 'Content-Type'
 
     # --------Static Routes-------
+    @self.app.route('/api')
+    @cross_origin()
+    def serve_root():
+      workdir, filename = os.path.split(os.path.abspath(__file__))
+      update = check_for_update()
+      root_dict = {
+        "version": {
+          "version": os.getenv('VERSION'),
+          "update_available": update.update_available if update is not None else None,
+          "up_to_date": update.up_to_date if update is not None else None,
+          "develop_version": update.development if update is not None else None,
+        },
+        "mode": os.getenv('MODE'),
+        "name": os.getenv('NAME'),
+        "info": {
+          "files_size_sum": self.get_file_size(),
+          "files": os.listdir(f'{workdir}/data')
+        }
+      }
+      return jsonify(root_dict)
+
+    @self.app.route('/api/json/<path:path>')
+    @cross_origin()
+    def serve_json(path):
+      return send_from_directory(f'{workdir}/data', path)
+
     @self.app.route('/charts')
     @cross_origin()
     def serve_index():
       return send_from_directory('/src', 'chart.html')
 
-    @self.app.route('/json')
+  # --------Helpers-------
-    @cross_origin()
+  def get_file_size(self):
-    def serve_get_list_of_json():
     workdir, filename = os.path.split(os.path.abspath(__file__))
-      return jsonify(os.listdir(f'{workdir}/data'))
+    files = os.listdir(f'{workdir}/data')
-
+    sizes = 0
-    @self.app.route('/json/<path:path>')
+    for file in files:
-    @cross_origin()
+      if file.endswith('.json'): sizes += os.path.getsize(f'{workdir}/data/{file}')
-    def serve_json(path):
+    return sizes
-      workdir, filename = os.path.split(os.path.abspath(__file__))
-      return send_from_directory(f'{workdir}/data', path)
 
 
 api = API()

python/src/ble_discovery.py

@@ -3,6 +3,8 @@ from bluepy.btle import Scanner
 
 from data_class import Data
 from devices import get_device
+from logger import get_logger
+logger = get_logger(__name__)
 
 # This is the list, where the responses will be stored from the `handleDiscovery`
 devices = []
@@ -38,9 +40,9 @@ class ScanDelegate(DefaultDelegate):
 
     device_from_config = get_device(dev)
 
-    try:print(f"Device: {dev.addr.upper()} ({dev.addrType}), RSSI: {dev.rssi}dB, Room: {device_from_config.room}")
+    try:logger.info(f"Device: {dev.addr.upper()} ({dev.addrType}), RSSI: {dev.rssi}dB, Room: {device_from_config.room}")
-    except:print(f"Device: {dev.addr.upper()} ({dev.addrType}), RSSI: {dev.rssi}dB, Room: ?")
+    except:logger.info(f"Device: {dev.addr.upper()} ({dev.addrType}), RSSI: {dev.rssi}dB, Room: ?")
-    print(f'\tTemp: {data_obj.temperature}°C, Humid: {data_obj.humidity}%, Batt: {data_obj.battery_percent}%\n')
+    logger.info(f'\tTemp: {data_obj.temperature}°C, Humid: {data_obj.humidity}%, Batt: {data_obj.battery_percent}%')
     return True
 
 
@@ -52,7 +54,7 @@ def cleanup():
 def start_discovery(timeout=20):
   cleanup()
   global devices
-  print(f'Start discovery with timout {timeout}s...')
+  logger.info(f'Start discovery with timout {timeout}s...')
 
   scanner = Scanner().withDelegate(ScanDelegate())
   scanner.scan(timeout=timeout, passive=False)

python/src/check_update.py

@@ -0,0 +1,74 @@
+import os
+import json
+import requests
+from logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class State:
+  def __init__(self, version_int:int):
+    self.version_int:int = version_int
+    self.version_str:str = f"{str(version_int)[0]}{str(version_int)[1]}-{str(version_int)[2]}{str(version_int)[3]}-{str(version_int)[4]}{str(version_int)[5]}"
+    self.update_available:bool = False
+    self.up_to_date:bool = False
+    self.development:bool = False
+
+
+class Release:
+  def __init__(self, data:dict):
+    self.name = data['name']
+    self.tag_name = data['tag_name']
+    self.description = data['description']
+    self.created_at = data['created_at']
+    self.released_at = data['released_at']
+    self.upcoming_release = data['upcoming_release']
+    self.version_int = int(self.tag_name.replace("-", ""))
+
+
+def check_for_update():
+  try: version_current = int(os.getenv('VERSION').replace("-", ""))
+  except:
+    logger.error("Error getting current version")
+    return
+  project_id = 58341398
+  request = f"https://gitlab.com/api/v4/projects/{project_id}/releases"
+  response = requests.get(url=request, timeout=1)
+  if not response.ok: return
+
+  releases_json = json.loads(response.text)
+  # Date, Object
+  latest = [0, None]
+  for release_json in releases_json:
+    release = Release(release_json)
+    if release.version_int > latest[0]:
+      latest[0] = release.version_int
+      latest[1] = release
+    logger.debug(repr(latest))
+
+  release = latest[1]
+  if release.version_int > version_current:
+    state = State(release.version_int)
+    state.update_available = True
+    return state
+
+  elif release.version_int == version_current:
+    state = State(release.version_int)
+    state.up_to_date = True
+    return state
+
+  else:
+    state = State(release.version_int)
+    state.development = True
+    return state
+
+
+def print_state(state:State):
+  if state is None: return
+  logger.info(f"Current version: {os.getenv('VERSION')}")
+  if state.update_available:
+    logger.info(f"Update available: {state.version_str}")
+  if state.up_to_date:
+    logger.info(f"Up to date")
+  if state.development:
+    logger.info(f"Development Version")

python/src/find_gateways.py

@@ -0,0 +1,77 @@
+# https://github.com/jholtmann/ip_discovery
+
+import os
+import FindMyIP
+from socket import *
+from helpers import get_unix_time
+from logger import get_logger
+
+DEBUG = True if os.environ.get('DEBUG') is not None else False
+DISCOVERY_ACK = 'IP_DISCOVERY_ACK'.encode()           # ACK for broadcast
+DISCOVERY_RSP_GTW = 'IP_DISCOVERY_RSP_GTW'.encode()   # RSP for gateway
+DISCOVERY_RSP_MSH = 'IP_DISCOVERY_RSP_MSH'.encode()   # RSP for mesh
+DISCOVERY_TIMEOUT = 1
+SOCKET_TIMEOUT = 0.2
+PORT_SERVER = 9434
+PORT_CLIENT = 9435
+
+
+def start_discovery_server():
+  logger = get_logger(__name__)
+  sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)
+  sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
+  sock.setsockopt(SOL_SOCKET, SO_BROADCAST, 1)
+  server_address = ('', PORT_SERVER)
+  local_ip = FindMyIP.internal()
+  try:
+    sock.bind(server_address)
+    logger.info("Started discovery socket")
+    while True:
+      data, addr = sock.recvfrom(4096)
+      logger.debug(f"Received a packet from {addr}")
+      logger.debug(f"{addr[0]} | {local_ip}")
+      logger.debug(f"{data} | {DISCOVERY_ACK}")
+
+      if data == DISCOVERY_ACK:
+        logger.debug("ACK accepted")
+        if str(addr[0]) == str(local_ip): continue
+        logger.debug("IP accepted")
+        sock.sendto(DISCOVERY_RSP_GTW, (addr[0], PORT_CLIENT))
+        logger.debug(f"Send ACK to {addr}")
+
+  except Exception as err:
+    logger.error(err)
+    sock.close()
+
+
+def start_discovery_client():
+  print("Started discovery client")
+  sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)
+  sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
+  sock.setsockopt(SOL_SOCKET, SO_BROADCAST, 1)
+  sock.settimeout(SOCKET_TIMEOUT)
+  server_address = ('255.255.255.255', PORT_SERVER)
+  start_time_stamp = get_unix_time()
+  delta = round(get_unix_time() - start_time_stamp, 2)
+  discovered_devices = []
+
+  try:
+    sock.bind(('', PORT_CLIENT))
+    while delta <= DISCOVERY_TIMEOUT:
+      delta = round(get_unix_time() - start_time_stamp, 2)
+      sock.sendto(DISCOVERY_ACK, server_address)
+      data, addr = sock.recvfrom(4096)
+      if data == DISCOVERY_RSP_GTW or data == DISCOVERY_RSP_MSH:
+        if str(addr[0]) in discovered_devices: continue
+        print('IP: ' + str(addr[0]))
+        discovered_devices.append(str(addr[0]))
+  except Exception as err:
+    print(err)
+
+  finally:
+    sock.close()
+    return discovered_devices
+
+
+# devices = start_discovery_client()
+# print(f"Devices: {devices}")

python/src/helpers.py

@@ -0,0 +1,16 @@
+import datetime
+import time
+
+
+def get_time_now():
+  date_now = datetime.datetime.now()
+  return str(date_now).split(" ")[0]
+
+
+def get_unix_time():
+  return time.time()
+
+
+def get_date():
+  date_now = datetime.datetime.now()
+  return str(date_now).split(" ")

python/src/log_data.py

@@ -3,8 +3,8 @@ import sys
 import json
 from data_class import Data
 from devices import Device
-
+from logger import get_logger
-DEBUG = True if os.getenv('DEBUG') == 'true' else False
+logger = get_logger(__name__)
 
 
 def log_to_json(devices):
@@ -15,7 +15,7 @@ def log_to_json(devices):
     data_obj: Data
     from_config: Device
     file_name = f'{workdir}/data/{str(data_obj.mac).replace(":", "-")}.json'
-    print(file_name) if DEBUG else {}
+    logger.debug(f"Save to {file_name}")
 
     try:
       with open(file_name, 'r') as file: data = json.load(file)
@@ -30,12 +30,12 @@ def log_to_json(devices):
       "battery_percent": data_obj.battery_percent,
       "battery_volt": data_obj.battery_volt,
       "rssi": dev.rssi,
-      "name": from_config.name,
+      "name": from_config.name if from_config is not None else "Unknown",
-      "room": from_config.room
+      "room": from_config.room if from_config is not None else "Unknown"
     }
     data.append(measurements)
 
-    print(measurements) if DEBUG else {}
+    logger.debug(measurements)
 
     with open(file_name, 'w') as file: file.write(json.dumps(data, indent=2))
 

python/src/logger.py

@@ -0,0 +1,21 @@
+import os
+import logging
+
+
+def get_logger(logger_name:str, log_file='gateway.log'):
+  logger_name = logger_name.replace('__', '')
+  debug_level = os.getenv('DEBUG').upper()
+  if debug_level not in ('CRITICAL', 'ERROR', 'WARNING', 'WARN', 'INFO', 'DEBUG', 'NOTSET', 'FATAL'):
+    print(f'Loglevel "{debug_level}" is not supported.')
+    exit(0)
+  logger = logging.getLogger(logger_name)
+  logger.setLevel(logging.getLevelName(debug_level))
+  handler = logging.FileHandler(filename=f'data/{log_file}', encoding='utf-8', mode='a')
+  formatter = logging.Formatter('%(asctime)s|%(levelname)s|%(name)s|:%(message)s')
+  handler.setFormatter(formatter)
+  stream_handler = logging.StreamHandler()
+  stream_handler.setFormatter(formatter)
+  logger.addHandler(stream_handler)
+  logger.addHandler(handler)
+  logger.info(f"Logger {logger_name} init done")
+  return logger

python/src/loop.py

@@ -1,10 +1,12 @@
 from time import sleep
 from log_data import log_to_json
-from discovery import start_discovery
+from ble_discovery import start_discovery
+from logger import get_logger
+logger = get_logger(__name__)
 
 
 def start_loop(interval=40, timeout=20):
-  print(f"Starting loop with interval {interval}s")
+  logger.info(f"Starting loop with interval {interval}s")
   while True:
     devices = start_discovery(timeout=timeout)
     log_to_json(devices)

python/src/main.py

@@ -1,26 +1,34 @@
 import os
-from discovery import start_discovery
+from logger import get_logger
+from ble_discovery import start_discovery
 from log_data import log_to_json
 from loop import start_loop
+from check_update import check_for_update
+from check_update import print_state
 
 INTERVAL = 40
 TIMEOUT = 20
-DOCKER = True if os.getenv('DOCKER') == 'true' else False
+DOCKER = os.getenv('DOCKER') == 'true'
-DEBUG = True if os.getenv('DEBUG') == 'true' else False
+DEBUG = os.getenv('DEBUG')
 interval = os.getenv('LOOP')
 timeout = os.getenv('TIMEOUT')
 
-if DEBUG:
+logger = get_logger(__name__)
-  print(f"INTERVAL: {INTERVAL}")
-  print(f"TIMEOUT: {TIMEOUT}")
-  print(f"interval: {interval}")
-  print(f"timeout: {timeout}")
-  print(f"DOCKER: {DOCKER}")
-  print(f"DEBUG: {DEBUG}")
-  print("")
 
-if DOCKER:
+logger.debug(f"INTERVAL: {INTERVAL}")
-  print("Running in docker")
+logger.debug(f"TIMEOUT: {TIMEOUT}")
+logger.debug(f"interval: {interval}")
+logger.debug(f"timeout: {timeout}")
+logger.debug(f"DOCKER: {DOCKER}")
+logger.debug(f"DEBUG: {DEBUG}")
+logger.debug(f"VERSION: {os.getenv('VERSION')}")
+
+update_state = check_for_update()
+print_state(update_state)
+
+try:
+  if DOCKER:
+    logger.info('Running in Docker')
 
     try:INTERVAL = int(interval)
     except:pass
@@ -31,5 +39,7 @@ if DOCKER:
     if interval is None: log_to_json(start_discovery(timeout=TIMEOUT))
     else:start_loop(INTERVAL, TIMEOUT)
 
-else:
+  else:
     start_loop(interval=40)
+except Exception as err:
+  logger.error(err)

python/src/mqtt.py

@@ -1 +1,15 @@
-# TODO
+import paho.mqtt.client as MQTT
+
+topic_root = "/atc_mithermometer_gateway"
+
+mqtt = MQTT.Client(mqtt.CallbackAPIVersion.VERSION2)
+mqtt.connect("192.168.178.140", 1883, 60)
+
+
+def publish_measurement(mac):
+  topic = f"{topic_root}/measurements/{mac}"
+  topic_temp = f"{topic}/temperature"
+  topic_humid = f"{topic}/humidity"
+  topic_battery = f"{topic}/battery"
+
+  mqtt.publish(topic_temp, )

python/src/start_discovery_server.py

@@ -0,0 +1,2 @@
+from find_gateways import start_discovery_server
+start_discovery_server()

run_gateway.sh

@@ -1,18 +1,10 @@
 TAG="latest"
 CONTAINER="dasmoorhuhn/atc-mithermometer-gateway"
 CONTAINER_NAME="ATC_MiThermometer_Gateway"
-VOLUME=data
+VOLUME=$(pwd)/data
 
-BACKGROUND=""
+HELP="Using any command line argument except -d bypasses the .env file\n\n
-TIME_ZONE=""
+USAGE: sh run_docker.sh [OPTIONS] \n
-INTERACTIVE=false
-BUILD=false
-API=false
-DEBUG=false
-LOOP="0"
-TIMEOUT="0"
-
-HELP="USAGE: sh run_docker.sh [OPTIONS] \n
 [ -d ] Run in Backgrund \n
 [ -t | --tag ] Set a docker tag. Default: latest \n
 [ -b | --build ] Build the image before running the container \n
@@ -20,6 +12,8 @@ HELP="USAGE: sh run_docker.sh [OPTIONS] \n
 [ -i | --interactive ] Start the container in interactive mode. That means, you can read the console output in real time and abort via STRG+C \n
 [ -a | --api ] Start with the API \n
 [ -v | --volume ] Set the volume, where the data from the gateway will be stored. Use relative path like /home/user/gateway/data \n
+[ -n | --name ] Set a custom name for this gateway \n
+[ -m2 | --mesh-gateway ] Set the mode to mesh gateway. This gateway is meant to expand the bluetooth radius. Default mode is main gateway \n
 [ -tz | --timezone ] Set the timezone. Default is Europe/Berlin \n
 [ -to | --timeout ] Set the timeout for the bluetooth scan. default is 20s \n
 [ -h | --help ] Get this dialog \n
@@ -31,12 +25,63 @@ if [ "$?" != 0 ]; then
   exit 1
 fi
 
+
+check_for_devices_config() {
+  if [ ! -f devices.yml ]; then
+    touch devices.yml
+    echo 'devices:
+  - mac: A4:C1:38:00:00:00
+    name: "my_room"
+    room: "my_room"' >> devices.yml
+  fi
+}
+
+
 docker_run() {
   sudo killall -9 bluetoothd > /dev/null 2>&1
   echo Killing old container...
   docker stop $CONTAINER_NAME > /dev/null 2>&1
   docker container rm $CONTAINER_NAME > /dev/null 2>&1
 
+  check_for_devices_config
+
+  if [ "$SKIP_ENV" = true ]; then
+    echo "Skip env file"
+    ENV_EXISTS=false
+
+    BACKGROUND=""
+    TIME_ZONE=""
+    NAME=""
+    INTERACTIVE=true
+    BUILD=true
+    API=true
+    DEBUG="INFO"
+    MODE="1"
+    LOOP="40"
+    TIMEOUT="20"
+  else
+    if [ -e .env ]
+    then
+      echo Loading .env file
+      export $(cat .env | xargs)
+      ENV_EXISTS=true
+    else
+      echo No env file found
+      ENV_EXISTS=false
+
+      BACKGROUND=""
+      TIME_ZONE=""
+      NAME=""
+      INTERACTIVE=true
+      BUILD=true
+      API=true
+      DEBUG="INFO"
+      MODE="1"
+      LOOP="40"
+      TIMEOUT="20"
+    fi
+  fi
+
   COMMAND="docker run $BACKGROUND"
   COMMAND="$COMMAND --cap-add=SYS_ADMIN"
   COMMAND="$COMMAND --cap-add=NET_ADMIN"
@@ -45,6 +90,11 @@ docker_run() {
   COMMAND="$COMMAND --restart=on-failure"
   COMMAND="$COMMAND --volume=/var/run/dbus/:/var/run/dbus/"
   COMMAND="$COMMAND --volume=$VOLUME:/src/data"
+  COMMAND="$COMMAND --volume=$PWD/devices.yml:/src/devices.yml"
+
+  if [ "$ENV_EXISTS" = true ]; then
+    COMMAND="$COMMAND --env-file .env"
+  fi
 
   if [ "$INTERACTIVE" = true ]; then
     COMMAND="$COMMAND --interactive"
@@ -73,7 +123,15 @@ docker_run() {
     COMMAND="$COMMAND --env API=$API"
   fi
 
-  if [ "$DEBUG" = true ]; then
+  if [ "$MODE" != 1 ]; then
+    COMMAND="$COMMAND --env MODE=$MODE"
+  fi
+
+  if [ "$NAME" != "" ]; then
+    COMMAND="$COMMAND --env NAME=$NAME"
+  fi
+
+  if [ "$DEBUG" = "DEBUG" ]; then
     COMMAND="$COMMAND --env DEBUG=$DEBUG"
     COMMAND="$COMMAND $CONTAINER:$TAG"
     echo
@@ -81,6 +139,7 @@ docker_run() {
     echo
     echo DEBUG MODE
   else
+    COMMAND="$COMMAND --env DEBUG=$DEBUG"
     COMMAND="$COMMAND $CONTAINER:$TAG"
   fi
 
@@ -93,44 +152,68 @@ docker_run() {
 
 while [ "$1" != "" ]; do
   case $1 in
+    -se | --skip-env-file )
+      SKIP_ENV=true
+      shift
+      ;;
     -d )
       BACKGROUND="-d"
       shift
       ;;
     --debug )
-      DEBUG=true
+      shift
+      DEBUG=$1
+      SKIP_ENV=true
       shift
       ;;
     -a | --api)
       API=true
+      SKIP_ENV=true
       shift
       ;;
     -b | --build )
       BUILD=true
+      SKIP_ENV=true
       shift
       ;;
     -v | --volume )
       shift
       VOLUME=$1
+      SKIP_ENV=true
+      shift
+      ;;
+    -n | --name )
+      shift
+      NAME=$1
+      SKIP_ENV=true
+      shift
+      ;;
+    -m2 | --mesh-gateway)
+      MODE=2
+      SKIP_ENV=true
       shift
       ;;
     -tz | --timezone )
       shift
       TIME_ZONE=$1
+      SKIP_ENV=true
       shift
       ;;
     -to | --timeout )
       shift
       TIMEOUT=$1
+      SKIP_ENV=true
       shift
       ;;
     -t | --tag )
       shift
       TAG=$1
+      SKIP_ENV=true
       shift
       ;;
     -l | --loop )
       shift
+      SKIP_ENV=true
       firstchar=`echo $1 | cut -c1-1`
       if [ "$firstchar" = "-" ]; then
         LOOP=0
@@ -143,6 +226,7 @@ while [ "$1" != "" ]; do
       ;;
     -i | --interactive )
       INTERACTIVE=true
+      SKIP_ENV=true
       shift
       ;;
     -h | --help )