diff --git a/configurations/nixos/defiant/configuration.nix b/configurations/nixos/defiant/configuration.nix index e109d45..9be3065 100644 --- a/configurations/nixos/defiant/configuration.nix +++ b/configurations/nixos/defiant/configuration.nix @@ -306,6 +306,10 @@ passwordFile = config.sops.secrets."services/paperless_password".path; }; + panoramax = { + enable = true; + }; + qbittorrent = { enable = true; mediaDir = "/srv/qbittorent"; diff --git a/modules/common-modules/pkgs/default.nix b/modules/common-modules/pkgs/default.nix index 28141c8..e608350 100644 --- a/modules/common-modules/pkgs/default.nix +++ b/modules/common-modules/pkgs/default.nix @@ -1,4 +1,8 @@ {pkgs, ...}: { + imports = [ + ./python + ]; + nixpkgs.overlays = [ (final: prev: { webtoon-dl = @@ -31,5 +35,9 @@ (final: prev: { sgblur = pkgs.python3.pkgs.callPackage ./sgblur.nix {}; }) + (final: prev: { + # Override h3 C library to version 4.3.0 + h3 = pkgs.callPackage ./h3-c-lib.nix {}; + }) ]; } diff --git a/modules/common-modules/pkgs/h3-c-lib.nix b/modules/common-modules/pkgs/h3-c-lib.nix new file mode 100644 index 0000000..2615d3c --- /dev/null +++ b/modules/common-modules/pkgs/h3-c-lib.nix @@ -0,0 +1,36 @@ +{ + lib, + stdenv, + fetchFromGitHub, + cmake, + doxygen, +}: +stdenv.mkDerivation rec { + pname = "h3"; + version = "4.3.0"; + + src = fetchFromGitHub { + owner = "uber"; + repo = "h3"; + rev = "v${version}"; + hash = "sha256-DUILKZ1QvML6qg+WdOxir6zRsgTvk+En6yjeFf6MQBg="; + }; + + nativeBuildInputs = [ + cmake + doxygen + ]; + + cmakeFlags = [ + "-DBUILD_SHARED_LIBS=ON" + "-DBUILD_TESTING=OFF" + ]; + + meta = with lib; { + homepage = "https://github.com/uber/h3"; + description = "Hexagonal hierarchical geospatial indexing system"; + license = licenses.asl20; + maintainers = []; + platforms = platforms.all; + }; +} diff --git a/modules/common-modules/pkgs/panoramax.nix b/modules/common-modules/pkgs/panoramax.nix index e2dad14..75b5e0e 100644 --- a/modules/common-modules/pkgs/panoramax.nix +++ b/modules/common-modules/pkgs/panoramax.nix @@ -10,8 +10,27 @@ authlib, sentry-sdk, python-dateutil, + dateparser, croniter, pydantic, + flask-cors, + flask-compress, + flask-babel, + flasgger, + yoyo-migrations, + psycopg, + psycopg-pool, + tzdata, + email-validator, + pydantic-extra-types, + python-multipart, + fs, + fs-s3fs, + geopic-tag-reader, + pygeofilter, + pygeoif, + rfeed, + geojson-pydantic, ... }: let pname = "geovisio"; @@ -42,8 +61,29 @@ in authlib sentry-sdk python-dateutil + dateparser croniter pydantic + flask-cors + flask-compress + flask-babel + flasgger + yoyo-migrations + psycopg + psycopg-pool + tzdata + email-validator + pydantic-extra-types + python-multipart + fs + fs-s3fs + geopic-tag-reader + pygeofilter + pygeoif + rfeed + geojson-pydantic + # Missing from nixpkgs - may need custom packages: + # flask-executor ]; # Skip tests as they may require network access or specific setup diff --git a/modules/common-modules/pkgs/python/default.nix b/modules/common-modules/pkgs/python/default.nix new file mode 100644 index 0000000..f69c512 --- /dev/null +++ b/modules/common-modules/pkgs/python/default.nix @@ -0,0 +1,18 @@ +{...}: { + nixpkgs.overlays = [ + (final: prev: { + python3 = prev.python3.override { + packageOverrides = pythonPrev: pythonFinal: { + h3 = pythonPrev.callPackage ./h3.nix {h3 = final.h3;}; + pygeofilter = pythonPrev.callPackage ./pygeofilter.nix {}; + pygeoif = pythonPrev.callPackage ./pygeoif.nix {}; + rfeed = pythonPrev.callPackage ./rfeed.nix {}; + pyexiv2 = pythonPrev.callPackage ./pyexiv2.nix {}; + geojson-pydantic = pythonPrev.callPackage ./geojson-pydantic.nix {}; + geopic-tag-reader = pythonPrev.callPackage ./geopic-tag-reader.nix {}; + }; + }; + python3Packages = final.python3.pkgs; + }) + ]; +} diff --git a/modules/common-modules/pkgs/python/geojson-pydantic.nix b/modules/common-modules/pkgs/python/geojson-pydantic.nix new file mode 100644 index 0000000..96ec6b5 --- /dev/null +++ b/modules/common-modules/pkgs/python/geojson-pydantic.nix @@ -0,0 +1,48 @@ +{ + lib, + fetchPypi, + buildPythonPackage, + flit-core, + pydantic, + geojson, + ... +}: let + pname = "geojson_pydantic"; + version = "2.0.0"; +in + buildPythonPackage { + inherit pname version; + + pyproject = true; + + src = fetchPypi { + inherit pname version; + hash = "sha256-ti6LRFAt0a1Ri19zkDWoGSSnb5gMvbOk6JFu+RO+JC4="; + }; + + build-system = [ + flit-core + ]; + + dependencies = [ + pydantic + geojson + ]; + + # Skip tests as they may require specific setup + doCheck = false; + + # Disable runtime dependencies check + dontCheckRuntimeDeps = true; + + # Basic imports check + pythonImportsCheck = ["geojson_pydantic"]; + + meta = with lib; { + description = "Pydantic models for GeoJSON objects"; + homepage = "https://github.com/developmentseed/geojson-pydantic"; + license = licenses.mit; + maintainers = []; + platforms = platforms.all; + }; + } diff --git a/modules/common-modules/pkgs/python/geopic-tag-reader.nix b/modules/common-modules/pkgs/python/geopic-tag-reader.nix new file mode 100644 index 0000000..bd8451f --- /dev/null +++ b/modules/common-modules/pkgs/python/geopic-tag-reader.nix @@ -0,0 +1,70 @@ +{ + lib, + fetchFromGitLab, + buildPythonPackage, + flit-core, + typer, + xmltodict, + timezonefinder, + pytz, + types-pytz, + types-python-dateutil, + rtree, + python-dateutil, + pyexiv2, + ... +}: let + pname = "geopic-tag-reader"; + version = "1.8.0"; +in + buildPythonPackage { + inherit pname version; + + pyproject = true; + + src = fetchFromGitLab { + owner = "panoramax"; + repo = "server/geo-picture-tag-reader"; + rev = version; + sha256 = "0lzf5xxxcdqmq28bpvgpkxf5jxmh2nawwa4rl4yg04bdsi16rf1j"; + }; + + build-system = [ + flit-core + ]; + + dependencies = [ + typer + xmltodict + pyexiv2 + timezonefinder + pytz + types-pytz + types-python-dateutil + rtree + ]; + + optional-dependencies = { + write-exif = [ + python-dateutil + types-python-dateutil + ]; + }; + + # Skip tests as they may require network access or specific setup + doCheck = false; + + # Disable runtime dependencies check as some dependencies might have issues + dontCheckRuntimeDeps = true; + + # Disable imports check initially to avoid dependency issues + pythonImportsCheck = []; + + meta = with lib; { + description = "GeoPic Tag Reader - Python library to read and write standardized metadata from geolocated pictures EXIF metadata"; + homepage = "https://gitlab.com/panoramax/server/geo-picture-tag-reader"; + license = licenses.mit; + maintainers = []; + platforms = platforms.all; + }; + } diff --git a/modules/common-modules/pkgs/python/h3.nix b/modules/common-modules/pkgs/python/h3.nix new file mode 100644 index 0000000..2dc3d26 --- /dev/null +++ b/modules/common-modules/pkgs/python/h3.nix @@ -0,0 +1,81 @@ +{ + autoPatchelfHook, + buildPythonPackage, + cmake, + cython, + fetchFromGitHub, + h3, + lib, + ninja, + numpy, + pytestCheckHook, + pytest-cov-stub, + scikit-build-core, + stdenv, +}: +buildPythonPackage rec { + pname = "h3"; + version = "4.3.1"; + pyproject = true; + + # pypi version does not include tests + src = fetchFromGitHub { + owner = "uber"; + repo = "h3-py"; + tag = "v${version}"; + hash = "sha256-zt7zbBgSp2P9q7mObZeQZpW9Szip62dAYdPZ2cGTmi4="; + }; + + dontConfigure = true; + + nativeCheckInputs = [ + pytestCheckHook + pytest-cov-stub + ]; + + build-system = + [ + scikit-build-core + cmake + cython + ninja + ] + ++ lib.optionals stdenv.hostPlatform.isLinux [ + # On Linux the .so files ends up referring to libh3.so instead of the full + # Nix store path. I'm not sure why this is happening! On Darwin it works + # fine. + autoPatchelfHook + ]; + + # This is not needed per-se, it's only added for autoPatchelfHook to work + # correctly. See the note above ^^ + buildInputs = lib.optionals stdenv.hostPlatform.isLinux [h3]; + + dependencies = [numpy]; + + # The following prePatch replaces the h3lib compilation with using the h3 packaged in nixpkgs. + # + # - Remove the h3lib submodule. + # - Patch CMakeLists to avoid building h3lib, and use h3 instead. + prePatch = let + cmakeCommands = '' + include_directories(${lib.getDev h3}/include/h3) + link_directories(${h3}/lib) + ''; + in '' + rm -r src/h3lib + substituteInPlace CMakeLists.txt \ + --replace-fail "add_subdirectory(src/h3lib)" "${cmakeCommands}" \ + --replace-fail "\''${CMAKE_CURRENT_BINARY_DIR}/src/h3lib/src/h3lib/include/h3api.h" "${lib.getDev h3}/include/h3/h3api.h" + ''; + + # Extra check to make sure we can import it from Python + pythonImportsCheck = ["h3"]; + + meta = { + homepage = "https://github.com/uber/h3-py"; + description = "Hierarchical hexagonal geospatial indexing system"; + license = lib.licenses.asl20; + maintainers = [lib.maintainers.kalbasit]; + }; +} diff --git a/modules/common-modules/pkgs/python/pyexiv2.nix b/modules/common-modules/pkgs/python/pyexiv2.nix new file mode 100644 index 0000000..69fa537 --- /dev/null +++ b/modules/common-modules/pkgs/python/pyexiv2.nix @@ -0,0 +1,49 @@ +{ + lib, + fetchFromGitHub, + buildPythonPackage, + exiv2, + boost, + pybind11, + setuptools, + ... +}: let + pname = "pyexiv2"; + version = "2.15.3"; +in + buildPythonPackage { + inherit pname version; + + pyproject = true; + build-system = [setuptools]; + + src = fetchFromGitHub { + owner = "LeoHsiao1"; + repo = "pyexiv2"; + rev = "v${version}"; + sha256 = "sha256-83bFMaoXncvhRJNcCgkkC7B29wR5pjuLO/EdkQdqxxo="; + }; + + buildInputs = [ + exiv2 + boost + ]; + + nativeBuildInputs = [ + pybind11 + ]; + + # Skip tests as they may require specific test images + doCheck = false; + + # Disable runtime dependencies check initially + dontCheckRuntimeDeps = true; + + meta = with lib; { + description = "Python binding to the library exiv2"; + homepage = "https://github.com/LeoHsiao1/pyexiv2"; + license = licenses.gpl3Plus; + maintainers = []; + platforms = platforms.linux; + }; + } diff --git a/modules/common-modules/pkgs/python/pygeofilter.nix b/modules/common-modules/pkgs/python/pygeofilter.nix new file mode 100644 index 0000000..aa310f9 --- /dev/null +++ b/modules/common-modules/pkgs/python/pygeofilter.nix @@ -0,0 +1,52 @@ +{ + lib, + fetchPypi, + buildPythonPackage, + setuptools, + wheel, + lark, + python-dateutil, + shapely, + ... +}: let + pname = "pygeofilter"; + version = "0.3.1"; +in + buildPythonPackage { + inherit pname version; + + pyproject = true; + + src = fetchPypi { + inherit pname version; + hash = "sha256-+SvAYiCZ+H/os23nq92GBZ1hWontYIInNwgiI6V44VA="; + }; + + build-system = [ + setuptools + wheel + ]; + + dependencies = [ + lark + python-dateutil + shapely + ]; + + # Skip tests as they may require specific setup + doCheck = false; + + # Disable runtime dependencies check + dontCheckRuntimeDeps = true; + + # Basic imports check + pythonImportsCheck = ["pygeofilter"]; + + meta = with lib; { + description = "A pure Python parser implementation of OGC filtering standards"; + homepage = "https://github.com/geopython/pygeofilter"; + license = licenses.mit; + maintainers = []; + platforms = platforms.all; + }; + } diff --git a/modules/common-modules/pkgs/python/pygeoif.nix b/modules/common-modules/pkgs/python/pygeoif.nix new file mode 100644 index 0000000..12b8b12 --- /dev/null +++ b/modules/common-modules/pkgs/python/pygeoif.nix @@ -0,0 +1,48 @@ +{ + lib, + fetchPypi, + buildPythonPackage, + setuptools, + wheel, + typing-extensions, + ... +}: let + pname = "pygeoif"; + version = "1.5.1"; +in + buildPythonPackage { + inherit pname version; + + pyproject = true; + + src = fetchPypi { + inherit pname version; + hash = "sha256-8nprah7Lh66swrUbzFnKeb5w7RKgEE3oYBR4shPdXYE="; + }; + + build-system = [ + setuptools + wheel + ]; + + dependencies = [ + typing-extensions + ]; + + # Skip tests as they may require specific setup + doCheck = false; + + # Disable runtime dependencies check + dontCheckRuntimeDeps = true; + + # Basic imports check + pythonImportsCheck = ["pygeoif"]; + + meta = with lib; { + description = "A basic implementation of the __geo_interface__"; + homepage = "https://github.com/cleder/pygeoif"; + license = licenses.lgpl21Plus; + maintainers = []; + platforms = platforms.all; + }; + } diff --git a/modules/common-modules/pkgs/python/rfeed.nix b/modules/common-modules/pkgs/python/rfeed.nix new file mode 100644 index 0000000..0be8ab9 --- /dev/null +++ b/modules/common-modules/pkgs/python/rfeed.nix @@ -0,0 +1,40 @@ +{ + lib, + fetchPypi, + buildPythonPackage, + setuptools, + python-dateutil, +}: +buildPythonPackage rec { + pname = "rfeed"; + version = "1.1.1"; + pyproject = true; + + src = fetchPypi { + inherit pname version; + hash = "sha256-qpUG8oZrdPWjItOUoUpjwZpoJcLZR1X/GdRt0eJDSBk="; + }; + + build-system = [ + setuptools + ]; + + dependencies = [ + python-dateutil + ]; + + # No tests available in the package + doCheck = false; + + pythonImportsCheck = [ + "rfeed" + ]; + + meta = with lib; { + description = "RSS feed generation library for Python"; + homepage = "https://pypi.org/project/rfeed/"; + license = licenses.mit; + maintainers = []; + platforms = platforms.all; + }; +} diff --git a/modules/nixos-modules/server/panoramax/panoramax.nix b/modules/nixos-modules/server/panoramax/panoramax.nix index aae7052..2af9982 100644 --- a/modules/nixos-modules/server/panoramax/panoramax.nix +++ b/modules/nixos-modules/server/panoramax/panoramax.nix @@ -3,43 +3,7 @@ lib, pkgs, ... -}: let - dbUrlConfigured = config.services.panoramax.database.url != null; - individualDbConfigured = lib.all (x: x != null) [ - config.services.panoramax.database.host - config.services.panoramax.database.port - config.services.panoramax.database.username - config.services.panoramax.database.password - config.services.panoramax.database.name - ]; - - envContent = '' - # Panoramax Configuration - FLASK_APP=geovisio - ${ - if dbUrlConfigured - then "DB_URL=${config.services.panoramax.database.url}" - else '' - DB_HOST=${config.services.panoramax.database.host} - DB_PORT=${toString config.services.panoramax.database.port} - DB_USERNAME=${config.services.panoramax.database.username} - DB_PASSWORD=${config.services.panoramax.database.password} - DB_NAME=${config.services.panoramax.database.name} - '' - } - ${lib.optionalString (config.services.panoramax.storage.fsUrl != null) "FS_URL=${config.services.panoramax.storage.fsUrl}"} - ${lib.optionalString (config.services.panoramax.infrastructure.nbProxies != null) "INFRA_NB_PROXIES=${toString config.services.panoramax.infrastructure.nbProxies}"} - ${lib.optionalString (config.services.panoramax.flask.secretKey != null) "FLASK_SECRET_KEY=${config.services.panoramax.flask.secretKey}"} - ${lib.optionalString (config.services.panoramax.flask.sessionCookieDomain != null) "FLASK_SESSION_COOKIE_DOMAIN=${config.services.panoramax.flask.sessionCookieDomain}"} - ${lib.optionalString (config.services.panoramax.api.pictures.licenseSpdxId != null) "API_PICTURES_LICENSE_SPDX_ID=${config.services.panoramax.api.pictures.licenseSpdxId}"} - ${lib.optionalString (config.services.panoramax.api.pictures.licenseUrl != null) "API_PICTURES_LICENSE_URL=${config.services.panoramax.api.pictures.licenseUrl}"} - ${lib.optionalString (config.services.panoramax.port != null) "PORT=${toString config.services.panoramax.port}"} - ${lib.optionalString (config.services.panoramax.sgblur.enable) "SGBLUR_API_URL=${config.services.panoramax.sgblur.url}"} - ${lib.concatStringsSep "\n" (lib.mapAttrsToList (name: value: "${name}=${value}") config.services.panoramax.extraEnvironment)} - ''; - - envFile = pkgs.writeText "panoramax.env" envContent; -in { +}: { options.services = { panoramax = { enable = lib.mkEnableOption "panoramax"; @@ -217,10 +181,145 @@ in { config = lib.mkIf config.services.panoramax.enable (lib.mkMerge [ { - # TODO: start panoramax service + # Create panoramax user and group + users.users.${config.services.panoramax.user} = { + isSystemUser = true; + group = config.services.panoramax.group; + home = "/var/lib/panoramax"; + createHome = true; + }; + + users.groups.${config.services.panoramax.group} = {}; + + # Ensure storage directory exists with correct permissions + systemd.tmpfiles.rules = [ + "d '${config.services.panoramax.settings.storage.fsUrl}' 0755 ${config.services.panoramax.user} ${config.services.panoramax.group} - -" + ]; + + systemd.services.panoramax-api = { + description = "Panoramax API server (self hosted map street view)"; + after = ["network.target" "postgresql.service"]; + wantedBy = ["multi-user.target"]; + + environment = + { + # Core Flask configuration + FLASK_APP = "geovisio"; + + # Database configuration + DB_HOST = config.services.panoramax.database.host; + DB_PORT = toString config.services.panoramax.database.port; + DB_USERNAME = config.services.panoramax.database.user; + DB_NAME = config.services.panoramax.database.name; + + # Storage configuration + FS_URL = config.services.panoramax.settings.storage.fsUrl; + + # Infrastructure configuration + INFRA_NB_PROXIES = toString config.services.panoramax.settings.infrastructure.nbProxies; + + # Application configuration + PORT = toString config.services.panoramax.port; + + # Python path to include the panoramax package + PYTHONPATH = "${config.services.panoramax.package}/${pkgs.python3.sitePackages}"; + } + // (lib.optionalAttrs (config.services.panoramax.settings.flask.secretKey != null) { + FLASK_SECRET_KEY = config.services.panoramax.settings.flask.secretKey; + }) + // (lib.optionalAttrs (config.services.panoramax.settings.flask.sessionCookieDomain != null) { + FLASK_SESSION_COOKIE_DOMAIN = config.services.panoramax.settings.flask.sessionCookieDomain; + }) + // (lib.optionalAttrs (config.services.panoramax.settings.api.pictures.licenseSpdxId != null) { + API_PICTURES_LICENSE_SPDX_ID = config.services.panoramax.settings.api.pictures.licenseSpdxId; + }) + // (lib.optionalAttrs (config.services.panoramax.settings.api.pictures.licenseUrl != null) { + API_PICTURES_LICENSE_URL = config.services.panoramax.settings.api.pictures.licenseUrl; + }) + // (lib.optionalAttrs config.services.sgblur.enable { + SGBLUR_API_URL = config.services.sgblur.url; + }) + // config.services.panoramax.settings.extraEnvironment; + + path = with pkgs; [ + (python3.withPackages (ps: with ps; [config.services.panoramax.package waitress])) + ]; + + serviceConfig = { + ExecStart = "${pkgs.python3.withPackages (ps: with ps; [config.services.panoramax.package waitress])}/bin/waitress-serve --port ${toString config.services.panoramax.port} --call geovisio:create_app"; + User = config.services.panoramax.user; + Group = config.services.panoramax.group; + WorkingDirectory = "/var/lib/panoramax"; + Restart = "always"; + RestartSec = 5; + + # Security hardening + PrivateTmp = true; + ProtectSystem = "strict"; + ProtectHome = true; + ReadWritePaths = [ + "/var/lib/panoramax" + config.services.panoramax.settings.storage.fsUrl + ]; + NoNewPrivileges = true; + PrivateDevices = true; + ProtectKernelTunables = true; + ProtectKernelModules = true; + ProtectControlGroups = true; + RestrictSUIDSGID = true; + RestrictRealtime = true; + RestrictNamespaces = true; + LockPersonality = true; + MemoryDenyWriteExecute = true; + SystemCallArchitectures = "native"; + }; + }; + + # Open firewall if requested + networking.firewall.allowedTCPPorts = lib.mkIf config.services.panoramax.openFirewall [ + config.services.panoramax.port + ]; } (lib.mkIf config.services.sgblur.enable { - # TODO: start sg blur config + # SGBlur service configuration + systemd.services.sgblur = { + description = "SGBlur face and license plate blurring service"; + after = ["network.target"]; + wantedBy = ["multi-user.target"]; + + path = with pkgs; [ + config.services.sgblur.package + python3 + python3Packages.waitress + ]; + + serviceConfig = { + ExecStart = "${pkgs.python3Packages.waitress}/bin/waitress-serve --host ${config.services.sgblur.host} --port ${toString config.services.sgblur.port} src.detect.detect_api:app"; + WorkingDirectory = "${config.services.sgblur.package}"; + Restart = "always"; + RestartSec = 5; + + # Basic security hardening + PrivateTmp = true; + ProtectSystem = "strict"; + ProtectHome = true; + NoNewPrivileges = true; + PrivateDevices = true; + ProtectKernelTunables = true; + ProtectKernelModules = true; + ProtectControlGroups = true; + RestrictSUIDSGID = true; + RestrictRealtime = true; + RestrictNamespaces = true; + LockPersonality = true; + MemoryDenyWriteExecute = true; + SystemCallArchitectures = "native"; + }; + }; + + networking.firewall.allowedTCPPorts = lib.mkIf config.services.panoramax.openFirewall [ + config.services.sgblur.port + ]; }) (lib.mkIf config.services.panoramax.database.createDB { services.postgresql = {