This commit is contained in:
2025-12-05 10:55:37 +01:00
commit 396e290394
1423 changed files with 825479 additions and 0 deletions

175
.devcontainer/Dockerfile Normal file
View File

@@ -0,0 +1,175 @@
# syntax=docker/dockerfile:1
FROM --platform=$BUILDPLATFORM docker.io/node:20-bookworm-slim as main-app
ARG DEBIAN_FRONTEND=noninteractive
# Buildx provided, must be defined to use though
ARG TARGETARCH
# Can be workflow provided, defaults set for manual building
ARG JBIG2ENC_VERSION=0.29
ARG QPDF_VERSION=11.9.0
ARG GS_VERSION=10.03.1
# Set Python environment variables
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
# Ignore warning from Whitenoise
PYTHONWARNINGS="ignore:::django.http.response:517" \
PNGX_CONTAINERIZED=1
#
# Begin installation and configuration
# Order the steps below from least often changed to most
#
# Packages need for running
ARG RUNTIME_PACKAGES="\
# General utils
curl \
# Docker specific
gosu \
# Timezones support
tzdata \
# fonts for text file thumbnail generation
fonts-liberation \
gettext \
ghostscript \
gnupg \
icc-profiles-free \
imagemagick \
# PostgreSQL
postgresql-client \
# MySQL / MariaDB
mariadb-client \
# OCRmyPDF dependencies
tesseract-ocr \
tesseract-ocr-eng \
tesseract-ocr-deu \
tesseract-ocr-fra \
tesseract-ocr-ita \
tesseract-ocr-spa \
unpaper \
pngquant \
jbig2dec \
# lxml
libxml2 \
libxslt1.1 \
# itself
qpdf \
# Mime type detection
file \
libmagic1 \
media-types \
zlib1g \
# Barcode splitter
libzbar0 \
poppler-utils \
htop \
sudo"
# Install basic runtime packages.
# These change very infrequently
RUN set -eux \
echo "Installing system packages" \
&& apt-get update \
&& apt-get install --yes --quiet --no-install-recommends ${RUNTIME_PACKAGES}
ARG PYTHON_PACKAGES="ca-certificates"
RUN set -eux \
echo "Installing python packages" \
&& apt-get update \
&& apt-get install --yes --quiet ${PYTHON_PACKAGES}
COPY --from=ghcr.io/astral-sh/uv:0.7.8 /uv /bin/uv
RUN set -eux \
&& echo "Installing pre-built updates" \
&& echo "Installing qpdf ${QPDF_VERSION}" \
&& curl --fail --silent --show-error --location \
--output libqpdf29_${QPDF_VERSION}-1_${TARGETARCH}.deb \
https://github.com/paperless-ngx/builder/releases/download/qpdf-${QPDF_VERSION}/libqpdf29_${QPDF_VERSION}-1_${TARGETARCH}.deb \
&& curl --fail --silent --show-error --location \
--output qpdf_${QPDF_VERSION}-1_${TARGETARCH}.deb \
https://github.com/paperless-ngx/builder/releases/download/qpdf-${QPDF_VERSION}/qpdf_${QPDF_VERSION}-1_${TARGETARCH}.deb \
&& dpkg --install ./libqpdf29_${QPDF_VERSION}-1_${TARGETARCH}.deb \
&& dpkg --install ./qpdf_${QPDF_VERSION}-1_${TARGETARCH}.deb \
&& echo "Installing Ghostscript ${GS_VERSION}" \
&& curl --fail --silent --show-error --location \
--output libgs10_${GS_VERSION}.dfsg-1_${TARGETARCH}.deb \
https://github.com/paperless-ngx/builder/releases/download/ghostscript-${GS_VERSION}/libgs10_${GS_VERSION}.dfsg-1_${TARGETARCH}.deb \
&& curl --fail --silent --show-error --location \
--output ghostscript_${GS_VERSION}.dfsg-1_${TARGETARCH}.deb \
https://github.com/paperless-ngx/builder/releases/download/ghostscript-${GS_VERSION}/ghostscript_${GS_VERSION}.dfsg-1_${TARGETARCH}.deb \
&& curl --fail --silent --show-error --location \
--output libgs10-common_${GS_VERSION}.dfsg-1_all.deb \
https://github.com/paperless-ngx/builder/releases/download/ghostscript-${GS_VERSION}/libgs10-common_${GS_VERSION}.dfsg-1_all.deb \
&& dpkg --install ./libgs10-common_${GS_VERSION}.dfsg-1_all.deb \
&& dpkg --install ./libgs10_${GS_VERSION}.dfsg-1_${TARGETARCH}.deb \
&& dpkg --install ./ghostscript_${GS_VERSION}.dfsg-1_${TARGETARCH}.deb \
&& echo "Installing jbig2enc" \
&& curl --fail --silent --show-error --location \
--output jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb \
https://github.com/paperless-ngx/builder/releases/download/jbig2enc-${JBIG2ENC_VERSION}/jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb \
&& dpkg --install ./jbig2enc_${JBIG2ENC_VERSION}-1_${TARGETARCH}.deb
# setup docker-specific things
# These change sometimes, but rarely
WORKDIR /usr/src/paperless/src/docker/
COPY [ \
"docker/rootfs/etc/ImageMagick-6/paperless-policy.xml", \
"./" \
]
RUN set -eux \
&& echo "Configuring ImageMagick" \
&& mv paperless-policy.xml /etc/ImageMagick-6/policy.xml
# Packages needed only for building a few quick Python
# dependencies
ARG BUILD_PACKAGES="\
build-essential \
git \
# https://www.psycopg.org/docs/install.html#prerequisites
libpq-dev \
# https://github.com/PyMySQL/mysqlclient#linux
default-libmysqlclient-dev \
pkg-config"
# hadolint ignore=DL3042
RUN --mount=type=cache,target=/root/.cache/uv,id=pip-cache \
set -eux \
&& echo "Installing build system packages" \
&& apt-get update \
&& apt-get install --yes --quiet ${BUILD_PACKAGES}
RUN set -eux \
&& npm update -g pnpm
# add users, setup scripts
# Mount the compiled frontend to expected location
RUN set -eux \
&& echo "Setting up user/group" \
&& groupmod --new-name paperless node \
&& usermod --login paperless --home /usr/src/paperless node \
&& usermod -s /bin/bash paperless \
&& echo "paperless ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers \
&& echo "Creating volume directories" \
&& mkdir --parents --verbose /usr/src/paperless/paperless-ngx/data \
&& mkdir --parents --verbose /usr/src/paperless/paperless-ngx/media \
&& mkdir --parents --verbose /usr/src/paperless/paperless-ngx/consume \
&& mkdir --parents --verbose /usr/src/paperless/paperless-ngx/export \
&& mkdir --parents --verbose /usr/src/paperless/paperless-ngx/.venv \
&& echo "Adjusting all permissions" \
&& chown --from root:root --changes --recursive paperless:paperless /usr/src/paperless
VOLUME ["/usr/src/paperless/paperless-ngx/data", \
"/usr/src/paperless/paperless-ngx/media", \
"/usr/src/paperless/paperless-ngx/consume", \
"/usr/src/paperless/paperless-ngx/export", \
"/usr/src/paperless/paperless-ngx/.venv"]

94
.devcontainer/README.md Normal file
View File

@@ -0,0 +1,94 @@
# Paperless-ngx Development Environment
## Overview
Welcome to the Paperless-ngx development environment! This setup uses VSCode DevContainers to provide a consistent and seamless development experience.
### What are DevContainers?
DevContainers are a feature in VSCode that allows you to develop within a Docker container. This ensures that your development environment is consistent across different machines and setups. By defining a containerized environment, you can eliminate the "works on my machine" problem.
### Advantages of DevContainers
- **Consistency**: Same environment for all developers.
- **Isolation**: Separate development environment from your local machine.
- **Reproducibility**: Easily recreate the environment on any machine.
- **Pre-configured Tools**: Include all necessary tools and dependencies in the container.
## DevContainer Setup
The DevContainer configuration provides up all the necessary services for Paperless-ngx, including:
- Redis
- Gotenberg
- Tika
Data is stored using Docker volumes to ensure persistence across container restarts.
## Configuration Files
The setup includes debugging configurations (`launch.json`) and tasks (`tasks.json`) to help you manage and debug various parts of the project:
- **Backend Debugging:**
- `manage.py runserver`
- `manage.py document-consumer`
- `celery`
- **Maintenance Tasks:**
- Create superuser
- Run migrations
- Recreate virtual environment (`.venv` with `uv`)
- Compile frontend assets
## Getting Started
### Step 1: Running the DevContainer
To start the DevContainer:
1. Open VSCode.
2. Open the project folder.
3. Open the command palette and choose `Dev Containers: Rebuild and Reopen in Container`.
VSCode will build and start the DevContainer environment.
### Step 2: Initial Setup
Once the DevContainer is up and running, run the `Project Setup: Run all Init Tasks` task to initialize the project.
Alternatively, the Project Setup can be done with individual tasks:
1. **Compile Frontend Assets**: `Maintenance: Compile frontend for production`.
2. **Run Database Migrations**: `Maintenance: manage.py migrate`.
3. **Create Superuser**: `Maintenance: manage.py createsuperuser`.
### Debugging and Running Services
You can start and debug backend services either as debugging sessions via `launch.json` or as tasks.
#### Using `launch.json`
1. Press `F5` or go to the **Run and Debug** view in VSCode.
2. Select the desired configuration:
- `Runserver`
- `Document Consumer`
- `Celery`
#### Using Tasks
1. Open the command palette and select `Tasks: Run Task`.
2. Choose the desired task:
- `Runserver`
- `Document Consumer`
- `Celery`
### Additional Maintenance Tasks
Additional tasks are available for common maintenance operations:
- **Recreate .venv**: For setting up the virtual environment using `uv`.
- **Migrate Database**: To apply database migrations.
- **Create Superuser**: To create an admin user for the application.
## Let's Get Started!
Follow the steps above to get your development environment up and running. Happy coding!

View File

@@ -0,0 +1,28 @@
{
"name": "Paperless Development",
"dockerComposeFile": "docker-compose.devcontainer.sqlite-tika.yml",
"service": "paperless-development",
"workspaceFolder": "/usr/src/paperless/paperless-ngx",
"postCreateCommand": "/bin/bash -c 'rm -rf .venv/.* && uv sync --group dev && uv run pre-commit install'",
"customizations": {
"vscode": {
"extensions": [
"mhutchie.git-graph",
"ms-python.python",
"ms-vscode.js-debug-nightly",
"eamodio.gitlens",
"yzhang.markdown-all-in-one"
],
"settings": {
"python.defaultInterpreterPath": "/usr/src/paperless/paperless-ngx/.venv/bin/python",
"python.pythonPath": "/usr/src/paperless/paperless-ngx/.venv/bin/python",
"python.terminal.activateEnvInCurrentTerminal": true,
"editor.formatOnPaste": false,
"editor.formatOnSave": true,
"editor.formatOnType": true,
"files.trimTrailingWhitespace": true
}
}
},
"remoteUser": "paperless"
}

View File

@@ -0,0 +1,77 @@
# Docker Compose file for developing Paperless NGX in VSCode DevContainers.
# This file contains everything Paperless NGX needs to run.
# Paperless supports amd64, arm, and arm64 hardware.
# All compose files of Paperless configure it in the following way:
#
# - Paperless is (re)started on system boot if it was running before shutdown.
# - Docker volumes for storing data are managed by Docker.
# - Folders for importing and exporting files are created in the same directory
# as this file and mounted to the correct folders inside the container.
# - Paperless listens on port 8000.
#
# SQLite is used as the database. The SQLite file is stored in the data volume.
#
# In addition, this Docker Compose file adds the following optional
# configurations:
#
# - Apache Tika and Gotenberg servers are started with Paperless NGX and Paperless
# is configured to use these services. These provide support for consuming
# Office documents (Word, Excel, PowerPoint, and their LibreOffice counterparts).
#
# This file is intended only to be used through VSCOde devcontainers. See README.md
# in the folder .devcontainer.
services:
broker:
image: docker.io/library/redis:7
restart: unless-stopped
volumes:
- ./redisdata:/data
# No ports need to be exposed; the VSCode DevContainer plugin manages them.
paperless-development:
image: paperless-ngx
build:
context: ../ # Dockerfile cannot access files from parent directories if context is not set.
dockerfile: ./.devcontainer/Dockerfile
restart: unless-stopped
depends_on:
- broker
- gotenberg
- tika
volumes:
- ..:/usr/src/paperless/paperless-ngx:delegated
- ../.devcontainer/vscode:/usr/src/paperless/paperless-ngx/.vscode:delegated # VSCode config files
- virtualenv:/usr/src/paperless/paperless-ngx/.venv # Virtual environment persisted in volume
- /usr/src/paperless/paperless-ngx/src/documents/static/frontend # Static frontend files exist only in container
- /usr/src/paperless/paperless-ngx/src/.pytest_cache
- /usr/src/paperless/paperless-ngx/.ruff_cache
- /usr/src/paperless/paperless-ngx/htmlcov
- /usr/src/paperless/paperless-ngx/.coverage
- ./data:/usr/src/paperless/paperless-ngx/data
- ./media:/usr/src/paperless/paperless-ngx/media
- ./consume:/usr/src/paperless/paperless-ngx/consume
environment:
PAPERLESS_REDIS: redis://broker:6379
PAPERLESS_TIKA_ENABLED: 1
PAPERLESS_TIKA_GOTENBERG_ENDPOINT: http://gotenberg:3000
PAPERLESS_TIKA_ENDPOINT: http://tika:9998
PAPERLESS_STATICDIR: ./src/documents/static
PAPERLESS_DEBUG: true
# Overrides default command so things don't shut down after the process ends.
command: /bin/sh -c "chown -R paperless:paperless /usr/src/paperless/paperless-ngx/src/documents/static/frontend && chown -R paperless:paperless /usr/src/paperless/paperless-ngx/.ruff_cache && while sleep 1000; do :; done"
gotenberg:
image: docker.io/gotenberg/gotenberg:8.17
restart: unless-stopped
# The Gotenberg Chromium route is used to convert .eml files. We do not
# want to allow external content like tracking pixels or even JavaScript.
command:
- "gotenberg"
- "--chromium-disable-javascript=true"
- "--chromium-allow-list=file:///tmp/.*"
tika:
image: docker.io/apache/tika:latest
restart: unless-stopped
volumes:
data:
media:
redisdata:
virtualenv:

View File

@@ -0,0 +1,58 @@
{
"version": "0.2.0",
"configurations": [
{
"name": "Chrome: Debug Angular Frontend",
"description": "Debug the Angular Dev Frontend in Chrome",
"type": "chrome",
"request": "launch",
"url": "http://localhost:4200",
"webRoot": "${workspaceFolder}/src-ui",
"preLaunchTask": "Start: Frontend Angular"
},
{
"name": "Debug: Backend Server (manage.py runserver)",
"description": "Debug the Django Backend Server",
"type": "python",
"request": "launch",
"program": "${workspaceFolder}/src/manage.py",
"args": [
"runserver"
],
"django": true,
"console": "integratedTerminal",
"env": {
"PYTHONPATH": "${workspaceFolder}/src"
},
"python": "${workspaceFolder}/.venv/bin/python"
},
{
"name": "Debug: Consumer Service (manage.py document_consumer)",
"description": "Debug the Consumer Service which processes files from a directory",
"type": "python",
"request": "launch",
"program": "${workspaceFolder}/src/manage.py",
"args": [
"document_consumer"
],
"django": true,
"console": "integratedTerminal",
"env": {
"PYTHONPATH": "${workspaceFolder}/src"
},
"python": "${workspaceFolder}/.venv/bin/python"
}
],
"compounds": [
{
"name": "Debug: FullStack",
"description": "Debug run the Angular dev frontend, Django backend, and consumer service",
"configurations": [
"Chrome: Debug Angular Frontend",
"Debug: Backend Server (manage.py runserver)",
"Debug: Consumer Service (manage.py document_consumer)"
],
"preLaunchTask": "Start: Celery Worker"
}
]
}

View File

@@ -0,0 +1,10 @@
{
"python.testing.pytestArgs": [],
"python.testing.unittestEnabled": false,
"python.testing.pytestEnabled": true,
"files.watcherExclude": {
"**/.venv/**": true,
"**/pytest_cache/**": true
},
"python.testing.cwd": "${workspaceFolder}/src"
}

View File

@@ -0,0 +1,223 @@
{
"version": "2.0.0",
"tasks": [
{
"label": "Start: Celery Worker",
"description": "Start the Celery Worker which processes background and consume tasks",
"type": "shell",
"command": "uv run celery --app paperless worker -l DEBUG",
"isBackground": true,
"options": {
"cwd": "${workspaceFolder}/src"
},
"problemMatcher": [
{
"owner": "custom",
"pattern": [
{
"regexp": ".",
"file": 1,
"location": 2,
"message": 3
}
],
"background": {
"activeOnStart": true,
"beginsPattern": "celery.*",
"endsPattern": "ready"
}
}
]
},
{
"label": "Start: Frontend Angular",
"description": "Start the Frontend Angular Dev Server",
"type": "shell",
"command": "pnpm start",
"isBackground": true,
"options": {
"cwd": "${workspaceFolder}/src-ui"
},
"problemMatcher": [
{
"owner": "custom",
"pattern": [
{
"regexp": ".",
"file": 1,
"location": 2,
"message": 3
}
],
"background": {
"activeOnStart": true,
"beginsPattern": ".*",
"endsPattern": "Compiled successfully"
}
}
]
},
{
"label": "Start: Consumer Service (manage.py document_consumer)",
"description": "Start the Consumer Service which processes files from a directory",
"type": "shell",
"command": "uv run python manage.py document_consumer",
"group": "build",
"presentation": {
"echo": true,
"reveal": "always",
"focus": false,
"panel": "shared",
"showReuseMessage": false,
"clear": true,
"revealProblems": "onProblem"
},
"options": {
"cwd": "${workspaceFolder}/src"
}
},
{
"label": "Start: Backend Server (manage.py runserver)",
"description": "Start the Backend Server which serves the Django API and the compiled Angular frontend",
"type": "shell",
"command": "uv run python manage.py runserver",
"group": "build",
"presentation": {
"echo": true,
"reveal": "always",
"focus": false,
"panel": "shared",
"showReuseMessage": false,
"clear": true,
"revealProblems": "onProblem"
},
"options": {
"cwd": "${workspaceFolder}/src"
}
},
{
"label": "Maintenance: manage.py migrate",
"description": "Apply database migrations",
"type": "shell",
"command": "uv run python manage.py migrate",
"group": "none",
"presentation": {
"echo": true,
"reveal": "always",
"focus": true,
"panel": "shared",
"showReuseMessage": false,
"clear": true,
"revealProblems": "onProblem"
},
"options": {
"cwd": "${workspaceFolder}/src"
}
},
{
"label": "Maintenance: Build Documentation",
"description": "Build the documentation with MkDocs",
"type": "shell",
"command": "uv run mkdocs build --config-file mkdocs.yml && uv run mkdocs serve",
"group": "none",
"presentation": {
"echo": true,
"reveal": "always",
"focus": true,
"panel": "shared",
"showReuseMessage": false,
"clear": true,
"revealProblems": "onProblem"
},
"options": {
"cwd": "${workspaceFolder}"
}
},
{
"label": "Maintenance: manage.py createsuperuser",
"description": "Create a superuser",
"type": "shell",
"command": "uv run python manage.py createsuperuser",
"group": "none",
"presentation": {
"echo": true,
"reveal": "always",
"focus": true,
"panel": "shared",
"showReuseMessage": false,
"clear": true,
"revealProblems": "onProblem"
},
"options": {
"cwd": "${workspaceFolder}/src"
}
},
{
"label": "Maintenance: recreate .venv",
"description": "Recreate the python virtual environment and install python dependencies",
"type": "shell",
"command": "rm -rf .venv && uv venv && uv sync --dev",
"group": "none",
"presentation": {
"echo": true,
"reveal": "always",
"focus": true,
"panel": "shared",
"showReuseMessage": false,
"clear": true,
"revealProblems": "onProblem"
},
"options": {
"cwd": "${workspaceFolder}"
}
},
{
"label": "Maintenance: Install Frontend Dependencies",
"description": "Install frontend (pnpm) dependencies",
"type": "pnpm",
"script": "install",
"path": "src-ui",
"group": "clean",
"problemMatcher": [],
"detail": "install dependencies from package"
},
{
"description": "Clean install frontend dependencies and build the frontend for production",
"label": "Maintenance: Compile frontend for production",
"type": "shell",
"command": "pnpm install && ./node_modules/.bin/ng build --configuration production",
"group": "none",
"presentation": {
"echo": true,
"reveal": "always",
"focus": true,
"panel": "shared",
"showReuseMessage": false,
"clear": true,
"revealProblems": "onProblem"
},
"options": {
"cwd": "${workspaceFolder}/src-ui"
}
},
{
"label": "Project Setup: Run all Init Tasks",
"description": "Runs all init tasks to setup the project including migrate the database, create a superuser and compile the frontend for production",
"dependsOrder": "sequence",
"dependsOn": [
"Maintenance: manage.py migrate",
"Maintenance: manage.py createsuperuser",
"Maintenance: Compile frontend for production"
]
},
{
"label": "Project Start: Run all Services",
"description": "Runs all services required to start the project including the Celery Worker, the Consumer Service and the Backend Server",
"dependsOn": [
"Start: Celery Worker",
"Start: Consumer Service (manage.py document_consumer)",
"Start: Backend Server (manage.py runserver)"
]
}
]
}