From 161525fbb5ea94d12f7159abab0e1ea16c59ad42 Mon Sep 17 00:00:00 2001 From: DasMoorhuhn Date: Sat, 25 May 2024 22:12:11 +0200 Subject: [PATCH] i18n, tests and more --- .coveragerc | 2 +- develop_requirements.txt | 6 ++-- install.sh | 1 + requirements.txt | 17 ++++----- src/config.py | 4 ++- src/config.yml | 7 +++- src/exif_data.py | 12 +++---- src/file_handler.py | 16 +++++---- src/i18n_translations/de.yml | 5 +++ src/i18n_translations/en.yml | 5 +++ src/meta_data_handler.py | 63 +++++++++++++++++++-------------- src/process.py | 23 ++++++++++-- tests/helpers/folder_helper.py | 4 +-- tests/start_tests.sh | 2 ++ tests/start_tests_gitlab.sh | 2 ++ tests/test_config.py | 5 ++- tests/tests/coverage/report.xml | 1 + 17 files changed, 117 insertions(+), 58 deletions(-) create mode 100644 install.sh create mode 100644 src/i18n_translations/de.yml create mode 100644 src/i18n_translations/en.yml create mode 100644 tests/tests/coverage/report.xml diff --git a/.coveragerc b/.coveragerc index 264914b..10aad34 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,3 +1,3 @@ [run] source = src -omit = tests/*, __init__.py, updater.py +omit = tests/*, __init__.py, updater.py, main.py diff --git a/develop_requirements.txt b/develop_requirements.txt index 8505e30..77e7772 100644 --- a/develop_requirements.txt +++ b/develop_requirements.txt @@ -1,3 +1,3 @@ -pytest==7.4.* -pytest-cov==4.1.* -pytest-factoryboy==2.5.* \ No newline at end of file +pytest +pytest-cov +pytest-factoryboy \ No newline at end of file diff --git a/install.sh b/install.sh new file mode 100644 index 0000000..0b5bb03 --- /dev/null +++ b/install.sh @@ -0,0 +1 @@ +sudo apt-get install -y python3 python3-pip \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 94b7ba4..53fcd6c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,9 @@ -pillow==10.1.* -pyyaml==6.0.* -python-magic==0.4.* -progressbar==2.5 -virtualenv==20.25.* -requests==2.31.* -filetype>=1.0.7 -rawpy==0.19.* \ No newline at end of file +pillow +pyyaml +python-magic +progressbar +virtualenv +requests +filetype +rawpy +python-i18n[YAML] \ No newline at end of file diff --git a/src/config.py b/src/config.py index bdb3c8d..c387c9b 100644 --- a/src/config.py +++ b/src/config.py @@ -1,7 +1,8 @@ import yaml import os -config_file = "config.yml" +workdir, filename = os.path.split(os.path.abspath(__file__)) +config_file = f"{workdir}{os.sep}config.yml" def get_config(): @@ -13,3 +14,4 @@ class Config: def __init__(self, data): self.src = data['src'] self.dst = data['dst'] + self.language = data['language'] diff --git a/src/config.yml b/src/config.yml index a0bfb57..6cfe118 100644 --- a/src/config.yml +++ b/src/config.yml @@ -1,2 +1,7 @@ +# src (source) is the path, were the images are, that should be sorted src: "../app/Temp" -dst: "../app/Bilder" \ No newline at end of file + +# dst (destination) ist the path, were the images are going to be sorted +dst: "../app/Bilder" + +language: "en" \ No newline at end of file diff --git a/src/exif_data.py b/src/exif_data.py index 471fac7..6136ed0 100644 --- a/src/exif_data.py +++ b/src/exif_data.py @@ -1,10 +1,10 @@ class ExifData: """This is for an object that stores the data of a picture""" def __init__(self, data:dict) -> None: - self.path:str = data['image_path'] - self.name:str = data['image_name'] - self.day:int = data['day'] - self.month:int = data['month'] - self.year = data['year'] + self.path:str = str(data['image_path']) + self.name:str = str(data['image_name']) + self.day:int = int(data['day']) + self.month:int = int(data['month']) + self.year:int = int(data['year']) self.time = data['time'] - self.make = data['make'] + self.make:str = str(data['make']) diff --git a/src/file_handler.py b/src/file_handler.py index 832c7b2..1be3dc0 100644 --- a/src/file_handler.py +++ b/src/file_handler.py @@ -1,7 +1,6 @@ import os import sys sys.path.append("../") -import time import shutil import logging from progressbar.progressbar import ProgressBar @@ -17,22 +16,25 @@ def sort_pictures(images:list, dst:str, logger:logging.Logger): term_width=70 ) - print(f"Start sorting {image_total} images\n") progress_bar.start() - start_timer = time.time() + for image in images: image:ExifData + if not image: continue path = os.path.join(dst, str(image.make), str(image.year), str(image.month), str(image.day)) if not os.path.exists(path): os.makedirs(path) stat_info = os.stat(image.path) - shutil.move(src=image.path, dst=f"{path}/{image.name}") + image_dst = f"{path}{os.sep}{image.name}" + shutil.move(src=image.path, dst=image_dst) # os.chmod(path=f"{path}/{image.name}", mode=stat_info.st_mode) - logger.info(f"Moved {image.path} -> {path}/{image.name}") + logger.info(f"Moved {image.path} -> {image_dst}") progress_bar.update(image_counter) image_counter += 1 - end_timer = time.time() progress_bar.finish() - print(f"\nDone\nSorted {image_total} images in {round(end_timer - start_timer, 2)} seconds") + + +def sort_raws(raws:list, dst:str, logger:logging): + pass diff --git a/src/i18n_translations/de.yml b/src/i18n_translations/de.yml new file mode 100644 index 0000000..43c95dc --- /dev/null +++ b/src/i18n_translations/de.yml @@ -0,0 +1,5 @@ +de: + start_sorting_images: 'Sortierung von %{image_count} Bild(ern) wird gestartet' + done_sorting_images: '%{image_count} Bild(er) in %{time} Sekunden sortiert' + done: 'Fertig' + no_images_found: 'Es wurden keine Bilder gefunden' \ No newline at end of file diff --git a/src/i18n_translations/en.yml b/src/i18n_translations/en.yml new file mode 100644 index 0000000..27e34ca --- /dev/null +++ b/src/i18n_translations/en.yml @@ -0,0 +1,5 @@ +en: + start_sorting_images: 'Start sorting %{image_count} image(s)\n' + done_sorting_images: 'Sorted %{image_count} image(s) in %{time} seconds' + done: 'Done' + no_images_found: 'No images found' \ No newline at end of file diff --git a/src/meta_data_handler.py b/src/meta_data_handler.py index 54ab19b..70a285a 100644 --- a/src/meta_data_handler.py +++ b/src/meta_data_handler.py @@ -1,5 +1,6 @@ import magic import sys +import os import filetype from PIL import Image from PIL import ExifTags @@ -9,7 +10,13 @@ from src.exif_data import ExifData from src.mime_types import MimeTypes video_formats = ["MP4", "MOV", "M4V", "MKV", "AVI", "WMV", "AVCHD", "WEBM", "MPEG"] -picture_formats = ["JPG", "JPEG", "PNG", "TIFF"] +picture_formats = ["JPG", "JPEG", "PNG"] +raw_formats = ["CR2", "RAF", "RW2", "ERF", "NRW", "NEF", "ARW", "RWZ", "EIP", + "DNG", "BAY", "DCR", "GPR", "RAW", "CRW", "3FR", "SR2", "K25", + "KC2", "MEF", "DNG", "CS1", "ORF", "MOS", "KDC", "CR3", "ARI", + "SRF", "SRW", "J6I", "FFF", "MRW", "MFW", "RWL", "X3F", "PEF", + "IIQ", "CXI", "NKSC", "MDC"] + key_words = ["DateTime", "Make"] @@ -27,34 +34,40 @@ def is_file_picture(path:str): else: return False -def get_image_meta_data(image_path): - # image_extension = str(image_path).split("/")[-1].split(".") - # TODO: Sort out videos - mime = MimeTypes(file_path=image_path) - if mime.is_video or mime.is_unsupported_file_type: return False +def handle_raw(image:str): + image_creation_time = os.path.getctime(filename=image) + print(image_creation_time) - img = Image.open(f"{image_path}") + +def handle_image(image:str): + img = Image.open(image) values = [] for tag, text in img.getexif().items(): if tag in ExifTags.TAGS: values.append([ExifTags.TAGS[tag], str(text)]) - return filter_date_and_make(meta_tags=filter_data(value=values), image_path=image_path) + return filter_date_and_make(meta_tags=filter_data(values=values), image_path=image) + + +def handle_video(video:str): + pass + + +def get_image_meta_data(image_path): + image_extension = str(image_path).split("/")[-1].split(".")[1].upper() + # TODO: Sort out videos using mime type of file + # mime = MimeTypes(file_path=image_path) + + if image_extension in picture_formats: return handle_image(image=image_path) + elif image_extension in video_formats: return handle_video(video=image_path) + elif image_extension in raw_formats: return handle_raw(image=image_path) def filter_date_and_make(meta_tags:list, image_path): - day = None - month = None - year = None - time = None - print(meta_tags) - make = meta_tags[1][1] + make = meta_tags[0][1] image_name = str(image_path).split("/")[-1] - _date = meta_tags[0] - time = _date[1].split(" ")[1] - _date = _date[1].split(" ")[0].split(":") - day = int(_date[2]) - month = int(_date[1]) - year = int(_date[0]) + date_time = meta_tags[1][1].split(" ") + date, time = date_time[0].split(":"), date_time[1] + year, month, day = date[0], date[1], date[2] exif_data_dict = { "day": day, @@ -69,13 +82,11 @@ def filter_date_and_make(meta_tags:list, image_path): return ExifData(exif_data_dict) -def filter_data(value): +def filter_data(values): + """Filters the data according to the meta tags from the keyword list""" value_return = [] - for k in key_words: - for v in value: - temp = v - if temp[0] == k: - value_return.append(v) + for value in values: + value_return.append(value) if value[0] in key_words else {} return value_return diff --git a/src/process.py b/src/process.py index fdba5f9..faa2418 100644 --- a/src/process.py +++ b/src/process.py @@ -1,22 +1,41 @@ +import os import sys +import i18n sys.path.append("../") +from time import time from src.meta_data_handler import get_meta_data from src.file_handler import sort_pictures from src.scan_folder import recursive_scan_folder from src.config import get_config +workdir, filename = os.path.split(os.path.abspath(__file__)) +config = get_config() + +i18n.set('locale', config.language) +i18n.set('fallback', 'en') +i18n.set('filename_format', '{locale}.{format}') +i18n.load_path.append(f'{workdir}{os.sep}i18n_translations{os.sep}') + def start_process(logger): - config = get_config() try: files = recursive_scan_folder(config.src) if len(files) > 0: + start_timer = time() exif_data = get_meta_data(images=files) + image_total = len(exif_data) + print(i18n.t('start_sorting_images', image_count=image_total)) + sort_pictures(images=exif_data, dst=config.dst, logger=logger) + end_timer = time() + duration = round(end_timer - start_timer, 2) + + print(i18n.t('done')) + print(i18n.t('done_sorting_images', time=duration, image_count=image_total)) return True else: - print("No images found") + print(i18n.t('no_images_found')) return False except Exception as err: print(err) diff --git a/tests/helpers/folder_helper.py b/tests/helpers/folder_helper.py index 609b1a7..536497f 100644 --- a/tests/helpers/folder_helper.py +++ b/tests/helpers/folder_helper.py @@ -5,7 +5,7 @@ from pathlib import Path from src.scan_folder import recursive_scan_folder TEST_FOLDER = ".test_folder" -TEST_IMAGES = "tests/test_files" +TEST_IMAGES = f"tests{os.sep}test_files" TEST_TEMP_FOLDER = os.path.join(TEST_FOLDER, 'Temp') TEST_IMAGE_FOLDER = os.path.join(TEST_FOLDER, 'Images') @@ -37,7 +37,7 @@ def copy_images(brand:str, model:str): create_folders() files = recursive_scan_folder(path=TEST_IMAGES) for file in files: - file_name = file.split("/")[2:][0] + file_name = file.split(os.sep)[2:][0] file_name = file_name.split("_") if file_name[0] == brand and file_name[1] == model: shutil.copy(src=file, dst=TEST_FOLDER) diff --git a/tests/start_tests.sh b/tests/start_tests.sh index 459ec60..c25bf8c 100755 --- a/tests/start_tests.sh +++ b/tests/start_tests.sh @@ -6,3 +6,5 @@ python3.12 -m pytest \ --cov-report xml:tests/coverage/coverage.xml \ --junitxml=tests/coverage/report.xml \ tests/ + +exit $? diff --git a/tests/start_tests_gitlab.sh b/tests/start_tests_gitlab.sh index 0a1239e..4abb642 100755 --- a/tests/start_tests_gitlab.sh +++ b/tests/start_tests_gitlab.sh @@ -3,8 +3,10 @@ pip3.12 install -r requirements.txt pip3.12 install -r develop_requirements.txt pwd sh tests/start_tests.sh +exit_code=$? cp tests/coverage/coverage.xml ./coverage.xml cp tests/coverage/report.xml ./report.xml python3.12 tests/get_coverage_percent.py +exit $exit_code \ No newline at end of file diff --git a/tests/test_config.py b/tests/test_config.py index c17e3de..76a2ca7 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -7,15 +7,18 @@ class TestConfig(unittest.TestCase): def test_config_class(self): test_config = { "src": "/src/src", - "dst": "/src/dst" + "dst": "/src/dst", + "language": "en" } config = config_module.Config(test_config) assert config.src == "/src/src" assert config.dst == "/src/dst" + assert config.language == "en" def test_get_config(self): config_module.config_file = "src/config.yml" config = config_module.get_config() assert config.src == "../app/Temp" assert config.dst == "../app/Bilder" + assert config.language == "en" diff --git a/tests/tests/coverage/report.xml b/tests/tests/coverage/report.xml new file mode 100644 index 0000000..4ef3cb0 --- /dev/null +++ b/tests/tests/coverage/report.xml @@ -0,0 +1 @@ + \ No newline at end of file