From 2242a46b314b84a79a794f68e36f5a6b2e6de5f8 Mon Sep 17 00:00:00 2001 From: Adrian Winterstein Date: Wed, 12 Mar 2025 18:00:38 +0100 Subject: [PATCH] Added Dockerfile and workflow to create and publish production containers. Co-authored-by: Saik0Shinigami --- .dockerignore | 4 + .github/actions/build-container/action.yml | 124 +++++++++++++++++++++ .github/workflows/build.yml | 28 +++++ .github/workflows/rebase.yml | 67 +++++++++++ .github/workflows/release.yml | 80 +++++++++++++ .github/workflows/test.yml | 9 +- Dockerfile | 101 +++++++++++++++++ README.md | 123 +++++++++++++++++++- docker-compose.yml | 55 ++++----- 9 files changed, 555 insertions(+), 36 deletions(-) create mode 100644 .github/actions/build-container/action.yml create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/rebase.yml create mode 100644 .github/workflows/release.yml create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore index 651665bbd9..270fd2800d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,6 @@ node_modules .git +.github +Dockerfile +docker-compose.* +README.md \ No newline at end of file diff --git a/.github/actions/build-container/action.yml b/.github/actions/build-container/action.yml new file mode 100644 index 0000000000..8d013aac5b --- /dev/null +++ b/.github/actions/build-container/action.yml @@ -0,0 +1,124 @@ +name: Container +description: Action for building the containers for a given release version +inputs: + version: + required: true + description: "The version tag for which the containers should be built" + push_containers: + required: false + default: "true" + description: "Whether the containers should be pushed to the registry after build" + create_release: + required: false + default: "true" + description: "Whether a release should be created after building the containers" + registry_user: + required: true + description: "User name for the container registry" + registry_token: + required: true + description: "Access token for the container registry" + release_access_token: + required: true + description: "Access token used to create a release" +runs: + using: "composite" + steps: + - uses: actions/checkout@v4 + if: inputs.create_release == 'true' + with: + ref: "releases/v${{ inputs.version }}" + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ inputs.registry_user }} + password: ${{ inputs.registry_token }} + - name: Determine major version + run: echo "version_major=$(echo ${{ inputs.version }} | cut -d. \-f1)" >> $GITHUB_OUTPUT + shell: bash + id: version_major + - name: Build and Push Server Develop + uses: docker/build-push-action@v6 + if: inputs.create_release != 'true' + with: + target: server + platforms: linux/amd64 + push: ${{ inputs.push_containers }} + tags: ${{ inputs.registry_user }}/habitica-server:${{ inputs.version }} + cache-from: type=registry,ref=${{ inputs.registry_user }}/habitica-server:buildcache + cache-to: type=registry,ref=${{ inputs.registry_user }}/habitica-server:buildcache,mode=max + - name: Build and Push Server Release + uses: docker/build-push-action@v6 + if: inputs.create_release == 'true' + with: + target: server + platforms: linux/amd64,linux/arm64 + push: ${{ inputs.push_containers }} + tags: ${{ inputs.registry_user }}/habitica-server:latest,${{ inputs.registry_user }}/habitica-server:${{ steps.version_major.outputs.version_major }},${{ inputs.registry_user }}/habitica-server:${{ inputs.version }} + cache-from: type=registry,ref=${{ inputs.registry_user }}/habitica-server:buildcache + cache-to: type=registry,ref=${{ inputs.registry_user }}/habitica-server:buildcache,mode=max + - name: Build and Push Client Develop + uses: docker/build-push-action@v6 + if: inputs.create_release != 'true' + with: + target: client + platforms: linux/amd64 + push: ${{ inputs.push_containers }} + tags: ${{ inputs.registry_user }}/habitica-client:${{ inputs.version }} + cache-from: type=registry,ref=${{ inputs.registry_user }}/habitica-server:buildcache + cache-to: type=registry,ref=${{ inputs.registry_user }}/habitica-server:buildcache,mode=max + - name: Build and Push Client Release + uses: docker/build-push-action@v6 + if: inputs.create_release == 'true' + with: + target: client + platforms: linux/amd64,linux/arm64 + push: ${{ inputs.push_containers }} + tags: ${{ inputs.registry_user }}/habitica-client:latest,${{ inputs.registry_user }}/habitica-client:${{ steps.version_major.outputs.version_major }},${{ inputs.registry_user }}/habitica-client:${{ inputs.version }} + cache-from: type=registry,ref=${{ inputs.registry_user }}/habitica-server:buildcache + cache-to: type=registry,ref=${{ inputs.registry_user }}/habitica-server:buildcache,mode=max + - name: Install zip + uses: montudor/action-zip@v1 + - name: Create Release Archives + if: inputs.create_release == 'true' + run: | + mkdir -p artifacts + + docker create --name server ${{ inputs.registry_user }}/habitica-server:${{ inputs.version }} + docker cp server:/var/lib/habitica artifacts/ + pushd artifacts/habitica + zip -r ../../habitica-server-v${{ inputs.version }}.zip * + popd + cp habitica-server-v${{ inputs.version }}.zip habitica-server-latest.zip + + docker create --name client ${{ inputs.registry_user }}/habitica-client:${{ inputs.version }} + docker cp client:/var/www artifacts/ + pushd artifacts/www + zip -r ../../habitica-client-v${{ inputs.version }}.zip * + popd + cp habitica-client-v${{ inputs.version }}.zip habitica-client-latest.zip + shell: bash + - name: Create release (latest) + if: inputs.create_release == 'true' + uses: "marvinpinto/action-automatic-releases@latest" + with: + repo_token: "${{ inputs.release_access_token }}" + prerelease: false + automatic_release_tag: "latest" + files: | + habitica-server-latest.zip + habitica-client-latest.zip + - name: Create release (version) + if: inputs.create_release == 'true' + uses: "marvinpinto/action-automatic-releases@latest" + with: + repo_token: "${{ inputs.release_access_token }}" + prerelease: false + automatic_release_tag: "v${{ inputs.version }}" + files: | + habitica-server-v${{ inputs.version }}.zip + habitica-client-v${{ inputs.version }}.zip diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000000..3a100a18b9 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,28 @@ +name: Container Build + +on: + push: + branches: + - self-host + workflow_dispatch: + +permissions: + contents: read + +jobs: + Build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v1 + with: + fetch-depth: 1 + - name: Build Container + uses: ./.github/actions/build-container + with: + version: "develop" + push_containers: "true" + create_release: "false" + registry_user: ${{ secrets.DOCKER_HUB_USER }} + registry_token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + release_access_token: ${{ secrets.ACTIONS_ACCESS_TOKEN }} diff --git a/.github/workflows/rebase.yml b/.github/workflows/rebase.yml new file mode 100644 index 0000000000..7745a0ee4d --- /dev/null +++ b/.github/workflows/rebase.yml @@ -0,0 +1,67 @@ +name: Rebase Upstream +on: + schedule: + - cron: "38 1 * * *" # run every night + workflow_dispatch: # allow manual running + +jobs: + Rebase: + runs-on: ubuntu-latest + outputs: + rebased: ${{ steps.rebase.outputs.rebased }} + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 1000 # greater than the number of commits made in the fork + + - name: Rebase on Upstream Release Branch + id: rebase + run: | + set -ex + + UPSTREAM="HabitRPG/habitica" + if [ -z $UPSTREAM ]; then + echo ${{ github.token }} | gh auth login --with-token + UPSTREAM=$(gh api repos/:owner/:repo --jq .parent.full_name) + if [ -z $UPSTREAM ]; then echo "Can't find upstream" >&2 && exit 1; fi + fi + if [ ! $(echo $UPSTREAM | egrep '^(http|git@)') ]; then + UPSTREAM=https://github.com/$UPSTREAM.git + fi + + git remote add upstream $UPSTREAM + + git fetch upstream release + + git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" + git config --local user.name "GitHub Actions" + + git rebase upstream/release + if [ "$(git status | grep diverged)" ]; then + git push origin $(git branch --show-current) --force-with-lease + echo "rebased=true" >> $GITHUB_OUTPUT + else + echo "rebased=false" >> $GITHUB_OUTPUT + fi + shell: bash + + Build: + needs: Rebase + if: needs.Rebase.outputs.rebased == 'true' + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + ref: self-host + fetch-depth: 1 + - name: Build Container + uses: ./.github/actions/build-container + with: + version: "develop" + push_containers: "false" + create_release: "false" + registry_user: ${{ secrets.DOCKER_HUB_USER }} + registry_token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + release_access_token: ${{ secrets.ACTIONS_ACCESS_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000000..c101f82e32 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,80 @@ +name: Create Release + +on: + schedule: + - cron: "20 3 * * *" # run every night + workflow_dispatch: # allow manual running + +jobs: + BranchAndRebase: + outputs: + version: ${{ steps.version.outputs.version }} + runs-on: ubuntu-latest + steps: + - name: Retrieve version + run: echo "version=$(git ls-remote --sort='-version:refname' --tags --refs https://github.com/HabitRPG/habitica.git | head -n1 | sed 's/.*\/v\?//')" >> $GITHUB_OUTPUT + id: version + - name: Check release branch exists already + run: | + if git ls-remote --sort='-version:refname' --refs https://github.com/awinterstein/habitica.git | grep "releases/v${{ steps.version.outputs.version }}"; then + echo "exists=true" >> $GITHUB_OUTPUT + else + echo "exists=false" >> $GITHUB_OUTPUT + fi + id: exists + - name: Checkout + if: steps.exists.outputs.exists != 'true' + uses: actions/checkout@v4 + with: + fetch-depth: 1000 # greater than the number of commits you made + - id: rebase + if: steps.exists.outputs.exists != 'true' + name: Rebase on upstream release branch + run: | + set -ex + + git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" + git config --local user.name "GitHub Actions" + + git remote add upstream https://github.com/HabitRPG/habitica.git + git fetch --tags upstream v${{ steps.version.outputs.version }} + + git checkout -b "releases/v${{ steps.version.outputs.version }}" + git rebase "v${{ steps.version.outputs.version }}" + + git push origin $(git branch --show-current) --force-with-lease + + shell: bash + + CheckContainerExistence: + outputs: + exists: ${{ steps.exists.outputs.exists }} + runs-on: ubuntu-latest + needs: BranchAndRebase + steps: + - name: Check if container exists already + run: | + if wget -q -O - "https://hub.docker.com/v2/namespaces/${{ secrets.DOCKER_HUB_USER }}/repositories/habitica-server/tags?page_size=10" | grep -o '"name": *"[^"]*' | grep -q ${{ needs.BranchAndRebase.outputs.version }} \ + && wget -q -O - "https://hub.docker.com/v2/namespaces/${{ secrets.DOCKER_HUB_USER }}/repositories/habitica-client/tags?page_size=10" | grep -o '"name": *"[^"]*' | grep -q ${{ needs.BranchAndRebase.outputs.version }}; then + echo "Container images exist already." + echo "exists=true" >> $GITHUB_OUTPUT + else + echo "exists=false" >> $GITHUB_OUTPUT + fi + id: exists + + BuildAndPushContainer: + runs-on: ubuntu-latest + needs: [BranchAndRebase, CheckContainerExistence] + if: needs.CheckContainerExistence.outputs.exists == 'false' + steps: + - uses: actions/checkout@v4 + - name: Build and push container + uses: ./.github/actions/build-container + with: + version: ${{ needs.BranchAndRebase.outputs.version }} + push_containers: "true" + create_release: "true" + registry_user: ${{ secrets.DOCKER_HUB_USER }} + registry_token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + release_access_token: ${{ secrets.ACTIONS_ACCESS_TOKEN }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 485c0095a4..f518fffb01 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,13 +1,6 @@ name: Test -on: - push: - branches-ignore: - - 'phillip/**' - - 'sabrecat/**' - - 'kalista/**' - - 'natalie/**' - pull_request: +on: [workflow_dispatch] # run manually] permissions: contents: read diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..15ac44f2b7 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,101 @@ +# Container stage for building server and web component of Habitica +FROM node:20 AS build + +ARG CI=true +ARG NODE_ENV=production + +RUN git config --global url."https://".insteadOf git:// + +WORKDIR /usr/src/habitica + +# Install main packages +COPY ["package.json", "package-lock.json", "./"] +RUN npm pkg set scripts.postinstall="echo \"Skipping postinstall\"" && npm install + +# Install client packages +COPY ["website/client/package.json", "website/client/package-lock.json", "./website/client/"] +RUN cd website/client/ && npm pkg set scripts.postinstall="echo \"Skipping postinstall\"" && npm install + +# Make the source code available in the container +COPY . /usr/src/habitica + +# Create configuration file (some values are needed for the client build already) +RUN echo '{\n\ + "BASE_URL": "http://localhost:3000",\n\ + "CRON_SAFE_MODE": "false",\n\ + "CRON_SEMI_SAFE_MODE": "false",\n\ + "DISABLE_REQUEST_LOGGING": "true",\n\ + "EMAIL_SERVER_AUTH_PASSWORD": "",\n\ + "EMAIL_SERVER_AUTH_USER": "",\n\ + "EMAIL_SERVER_URL": null,\n\ + "ENABLE_CONSOLE_LOGS_IN_PROD": "true",\n\ + "ENABLE_CONSOLE_LOGS_IN_TEST": "false",\n\ + "FLAG_REPORT_EMAIL": "",\n\ + "IGNORE_REDIRECT": "true",\n\ + "INVITE_ONLY": "false",\n\ + "MAINTENANCE_MODE": "false",\n\ + "MONGODB_POOL_SIZE": "10",\n\ + "NODE_ENV": "production",\n\ + "PATH": "bin:node_modules/.bin:/usr/local/bin:/usr/bin:/bin",\n\ + "PORT": 3000,\n\ + "PUSH_CONFIGS_APN_ENABLED": "false",\n\ + "SESSION_SECRET": "YOUR SECRET HERE",\n\ + "SESSION_SECRET_IV": "12345678912345678912345678912345",\n\ + "SESSION_SECRET_KEY": "1234567891234567891234567891234567891234567891234567891234567891",\n\ + "TRUSTED_DOMAINS": "",\n\ + "WEB_CONCURRENCY": 1,\n\ + "ENABLE_STACKDRIVER_TRACING": "false",\n\ + "BLOCKED_IPS": "",\n\ + "LOG_AMPLITUDE_EVENTS": "false",\n\ + "RATE_LIMITER_ENABLED": "false",\n\ + "CONTENT_SWITCHOVER_TIME_OFFSET": 8\n\ +}' > /usr/src/habitica/config.json + +# Build the server and web components +RUN ./node_modules/.bin/gulp build:prod +RUN npm run client:build + + + +# Container for providing the build server component of Habitica +FROM node:20 AS server + +ENV NODE_ENV=production + +COPY --from=build /usr/src/habitica/node_modules /var/lib/habitica/node_modules + +COPY --from=build /usr/src/habitica/i18n_cache/ /var/lib/habitica/i18n_cache/ +COPY --from=build /usr/src/habitica/content_cache/ /var/lib/habitica/content_cache/ + +COPY --from=build /usr/src/habitica/website/ /var/lib/habitica/website/ + +COPY --from=build /usr/src/habitica/package.json /var/lib/habitica/package.json +COPY --from=build /usr/src/habitica/config.json /var/lib/habitica/config.json + +CMD ["node", "/var/lib/habitica/website/transpiled-babel/index.js"] + + + +# Container for providing the build web component of Habitica +FROM caddy AS client + +COPY --from=build /usr/src/habitica/website/client/dist /var/www + +RUN echo -e ":80 {\n\ + @backend not {\n\ + path /static/audio/\n\ + path /static/css/\n\ + path /static/emails/\n\ + path /static/icons/\n\ + path /static/img/\n\ + path /static/js/\n\ + path /static/merch/\n\ + path /static/npc/\n\ + path /static/presskit/\n\ + path /index.html\n\ + }\n\ +\n\ + root * /var/www\n\ + reverse_proxy @backend server:3000\n\ + file_server\n\ +}" > /etc/caddy/Caddyfile \ No newline at end of file diff --git a/README.md b/README.md index caf1c4e0c0..ecfdc18d4d 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,124 @@ -Habitica ![Build Status](https://github.com/HabitRPG/habitica/workflows/Test/badge.svg) -=============== +# Habitica Self-Hosted + +Adaptions and infrastructure to facilitate self-hosting of the habit-building program [Habitica](https://habitica.com). It is based on the source code and assets of the [Habitica Repository](https://github.com/HabitRPG/habitica), hence the [LICENSE](https://github.com/HabitRPG/habitica/blob/develop/LICENSE) from there applies here and to the adaptions in this repository as well. + +![Screenshot of the Habitica Web Client](website/client/public/static/presskit/Samples/Website/Market.png) + +For each release in the Habitica upstream repository, the self-hosting adaptions are automatically applied by rebasing the `self-host` branch onto the last release commit. The Docker images for server and client are built then and pushed to Docker Hub as [awinterstein/habitica-server](https://hub.docker.com/r/awinterstein/habitica-server) and [awinterstein/habitica-client](https://hub.docker.com/r/awinterstein/habitica-client). + +## Improvements for Self-Hosting + +The following noteworthy changes were applied to the Habitica source code: + +- Dockerfile and Github Workflow to create the production containers for hosting +- every user automatically gets a subscription on registration that never needs to be renewed +- locations for buying gems with money were replaced with options to buy with gold (e.g., the quick access in the header) +- group plans can be created without payment +- emails are sent directly via a configured SMTP server instead of using the Mailchimp (Mandrill) web-service +- settings and links (e.g., in the footer) that do not make sense for a self-hosted site were removed +- registrations can be restricted to only invited users via a configuration parameter +- analytics and payment scripts are not loaded + +## Limitations + +The following things do not work (yet): +- third-party access and scripts are not thoroughly disabled, so there might still be some scripts loaded + +Contributions to fix those or other things are very welcome! + +## Simple Setup with Docker Compose + +Habitica needs a Mongo database, its server component (a NodeJS application) and its client component (a Vue.js application). In the simplest setup for self-hosting, there are two containers started for them, with a dependency from the server (that provides the server and the client component) to the database: + +```mermaid +architecture-beta + group containers(server)[Containers] + + service db(database)[MongoDB] in containers + service server(server)[Server] in containers + + db:L <-- R:server + + group host(server)[Host] + service proxy(server)[Reverse Proxy] in host + + proxy{group}:T --> B:server{group} +``` + +The server port could directly be exposed as port 80 on the host. However, usually a reverse proxy like Nginx would be put in front, that handles HTTPS traffic including TLS certificate handling. + +The following Docker Compose file can be used for setting up the containers: + +```yaml +version: "3" +services: + server: + image: docker.io/awinterstein/habitica-server:latest + restart: unless-stopped + depends_on: + - mongo + environment: + - NODE_DB_URI=mongodb://mongo/habitica # this only needs to be adapted if using a separate database + - BASE_URL=http://127.0.0.1:3000 # change this to the URL under which your instance will be reachable + - INVITE_ONLY=false # change to `true` after registration of initial users, to restrict further registrations + - EMAIL_SERVER_URL=mail.example.com + - EMAIL_SERVER_PORT=587 + - EMAIL_SERVER_AUTH_USER=mail_user + - EMAIL_SERVER_AUTH_PASSWORD=mail_password + ports: + - "3000:3000" + networks: + - habitica + mongo: + image: docker.io/mongo:latest # better to replace 'latest' with the concrete mongo version (e.g., the most recent one) + restart: unless-stopped + hostname: mongo + command: ["--replSet", "rs", "--bind_ip_all", "--port", "27017"] + healthcheck: + test: echo "try { rs.status() } catch (err) { rs.initiate() }" | mongosh --port 27017 --quiet + interval: 10s + timeout: 30s + start_period: 0s + start_interval: 1s + retries: 30 + volumes: + - ./db:/data/db:rw + - ./dbconf:/data/configdb + networks: + habitica: + aliases: + - mongo +networks: + habitica: + driver: bridge +``` + +> [!IMPORTANT] +> If you are planning to run the Habitica containers on a Raspberry Pi 4, you might not be able to use `mongo:latest` (see [issue 20](https://github.com/awinterstein/habitica/issues/20)). In this case you can try to use `mongo:bionic` instead. + +## Optimized Setup with Docker Compose + +As there's probably a web server running on the host already, acting as a reverse proxy for Habitica, this web server could be used to sever the static client files for Habitica as well. + +```mermaid +architecture-beta + group containers(server)[Containers] + service db(database)[MongoDB] in containers + service server(server)[Server] in containers + db:L <-- R:server + + group host(server)[Host] + service proxy(server)[Reverse Proxy] in host + proxy{group}:T --> B:server{group} + service client(database)[Client Files] in host + proxy:R --> L:client +``` + +Or the static client files could be served from a different host (e.g., a static file hosting). + +## Readme of the Upstream Habitica Repository + +![Build Status](https://github.com/HabitRPG/habitica/workflows/Test/badge.svg) [Habitica](https://habitica.com) is an open-source habit-building program that treats your life like a role-playing game. Level up as you succeed, lose HP as you fail, and earn Gold to buy weapons and armor! diff --git a/docker-compose.yml b/docker-compose.yml index 8ef78d4eea..33b1ef55f9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,36 +1,39 @@ version: "3" services: - - client: - build: . - networks: - - habitica - environment: - - BASE_URL=http://server:3000 - ports: - - "8080:8080" - command: ["npm", "run", "client:dev"] - depends_on: - - server - server: - build: . - ports: - - "3000:3000" - networks: - - habitica - environment: - - NODE_DB_URI=mongodb://mongo/habitrpg + build: + context: . + dockerfile: ./Dockerfile + target: server depends_on: - mongo - - mongo: - image: mongo:3.6 - ports: - - "27017:27017" + environment: + - NODE_DB_URI=mongodb://mongo/habitica # this only needs to be adapted if using a separate database + - BASE_URL=http://127.0.0.1:8080 # change this to the URL under which your instance will be reachable + - INVITE_ONLY=false # change to `true` after registration of initial users, to restrict further registrations networks: - habitica - + ports: + - "3000:3000" + mongo: + image: docker.io/mongo:latest + restart: unless-stopped + hostname: mongo + command: ["--replSet", "rs", "--bind_ip_all", "--port", "27017"] + healthcheck: + test: echo "try { rs.status() } catch (err) { rs.initiate() }" | mongosh --port 27017 --quiet + interval: 10s + timeout: 30s + start_period: 0s + start_interval: 1s + retries: 30 + volumes: + - ./mongodb-data/db:/data/db:rw,Z + - ./mongodb-data/dbconf:/data/configdb:Z + networks: + habitica: + aliases: + - mongo networks: habitica: driver: bridge