Merge pull request #23 from YunoHost-Apps/testing

urlwatch

README.md

@@ -53,7 +53,7 @@ Generally the [wiki](https://codeberg.org/flohmarkt/flohmarkt/wiki) is a good so
 * [Service compatibility chart](https://codeberg.org/flohmarkt/flohmarkt/wiki/Service-compatibility-chart)
 
 
-**Shipped version:** 0.0~ynh7
+**Shipped version:** 0.0~ynh8
 
 **Demo:** <https://flohmarkt.ween.de/>
 

README_eu.md

@@ -53,7 +53,7 @@ Generally the [wiki](https://codeberg.org/flohmarkt/flohmarkt/wiki) is a good so
 * [Service compatibility chart](https://codeberg.org/flohmarkt/flohmarkt/wiki/Service-compatibility-chart)
 
 
-**Paketatutako bertsioa:** 0.0~ynh7
+**Paketatutako bertsioa:** 0.0~ynh8
 
 **Demoa:** <https://flohmarkt.ween.de/>
 

README_fr.md

@@ -53,7 +53,7 @@ Generally the [wiki](https://codeberg.org/flohmarkt/flohmarkt/wiki) is a good so
 * [Service compatibility chart](https://codeberg.org/flohmarkt/flohmarkt/wiki/Service-compatibility-chart)
 
 
-**Version incluse :** 0.0~ynh7
+**Version incluse :** 0.0~ynh8
 
 **Démo :** <https://flohmarkt.ween.de/>
 

README_gl.md

@@ -53,7 +53,7 @@ Generally the [wiki](https://codeberg.org/flohmarkt/flohmarkt/wiki) is a good so
 * [Service compatibility chart](https://codeberg.org/flohmarkt/flohmarkt/wiki/Service-compatibility-chart)
 
 
-**Versión proporcionada:** 0.0~ynh7
+**Versión proporcionada:** 0.0~ynh8
 
 **Demo:** <https://flohmarkt.ween.de/>
 

README_zh_Hans.md

@@ -53,7 +53,7 @@ Generally the [wiki](https://codeberg.org/flohmarkt/flohmarkt/wiki) is a good so
 * [Service compatibility chart](https://codeberg.org/flohmarkt/flohmarkt/wiki/Service-compatibility-chart)
 
 
-**分发版本：** 0.0~ynh7
+**分发版本：** 0.0~ynh8
 
 **演示：** <https://flohmarkt.ween.de/>
 

conf/urlwatch.cron

@@ -0,0 +1,9 @@
+#!/bin/sh
+# run urlwatch for flohmarkt
+PATH=/usr/sbin:/sbin:/usr/bin:/bin
+
+sleep $((RANDOM%111)) 
+sudo -u __APP__ urlwatch \
+    --config /var/www/__APP__/urlwatch/config.yaml \
+    --urls=/var/www/__APP__/urlwatch/urls.yaml \
+    --cache /var/www/__APP__/urlwatch/cache.file

conf/urlwatch_config.yaml

@@ -0,0 +1,90 @@
+display:
+  error: true
+  new: true
+  unchanged: false
+job_defaults:
+  all: {}
+  browser: {}
+  shell: {}
+  url: {}
+report:
+  discord:
+    embed: false
+    enabled: false
+    max_message_length: 2000
+    subject: '{count} changes: {jobs}'
+    webhook_url: ''
+  email:
+    enabled: true
+    from: '__MAIL_USER__@__MAIL_DOMAIN__'
+    html: false
+    method: smtp
+    sendmail:
+      path: sendmail
+    smtp:
+      host: localhost
+      auth: true
+      insecure_password: '__MAIL_PWD__'
+      port: 25
+      starttls: true
+      user: '__APP__'
+    subject: 'New or deleted flohmarkt users'
+    to: '__ADMIN_MAIL__'
+  html:
+    diff: unified
+  ifttt:
+    enabled: false
+    event: ''
+    key: ''
+  mailgun:
+    api_key: ''
+    domain: ''
+    enabled: false
+    from_mail: ''
+    from_name: ''
+    region: us
+    subject: '{count} changes: {jobs}'
+    to: ''
+  markdown:
+    details: true
+    footer: true
+    minimal: false
+  matrix:
+    access_token: ''
+    enabled: false
+    homeserver: ''
+    room_id: ''
+  mattermost:
+    enabled: false
+    max_message_length: 40000
+    webhook_url: ''
+  pushbullet:
+    api_key: ''
+    enabled: false
+  pushover:
+    app: ''
+    device: null
+    enabled: false
+    priority: normal
+    sound: spacealarm
+    user: ''
+  slack:
+    enabled: false
+    max_message_length: 40000
+    webhook_url: ''
+  stdout:
+    color: true
+    enabled: true
+  telegram:
+    bot_token: ''
+    chat_id: ''
+    enabled: false
+  text:
+    details: true
+    footer: true
+    line_length: 75
+    minimal: false
+  xmpp:
+    enabled: false
+    recipient: ''
+    sender: ''

conf/urlwatch_urls.yaml

@@ -0,0 +1,11 @@
+name: "watch for changes in user database"
+kind: "url"
+url: "http://__APP__:__PASSWORD_COUCHDB_FLOHMARKT__@127.0.0.1:5984/flohmarkt/_find"
+method: "POST"
+data: '{"selector": { "role": "User", "name": { "$ne": "instance" }}, 
+  "fields": ["name","email","active"], "sort": [ "name" ]}'
+headers: 
+  Content-Type: application/json
+  Accept: application/json
+filter:
+  - shellpipe: 'jq .docs'

config_panel.toml

@@ -0,0 +1,33 @@
+version = "1.0"
+[main]
+    name.en = "configuration"
+    name.de = "Konfiguration"
+    [main.notifications]
+        name.en = "email notifications"
+        name.de = "E-Mail Benachrichtigungen"
+        [main.notifications.admin_mail]
+        type = "email"
+        ask.en = "recipient of notifications"
+        help.en = "this email gets notified about events like changes in the user database of flohmarkt"
+        ask.de = "Empfänger von Benachrichtigungen"
+        help.de = "an diese E-Mail Adresse werden Informationen zu z.B. Änderungen an der Benutzerdatenbank geschickt"
+        bind = "email>to:__INSTALL_DIR__/urlwatch/config.yaml"
+
+    [main.flohmarkt]
+        name.en = "flohmarkt settings"
+        name.de = "flohmarkt Konfiguration"
+        [main.flohmarkt.flohmarkt_initial_setup_url]
+        type = "text"
+        readonly = true
+        ask.en = "initial setup"
+        ask.de = "Initiales Setup"
+        help.en = "open this url to finish your flohmarkt setup"
+        help.de = "öffne die URL um flohmarkt fertig zu konfigurieren"
+
+#[flohmarkt_setup]
+#    name = "setup"
+#    [flohmarkt_setup.flohmarkt_initial_setup_url]
+#        name = "initial setup url"
+#        type = "string"
+#        ask = "open this url to initially setup flohmarkt"
+#        help = "you can't change this value"

doc/POST_INSTALL.md

@@ -1,13 +1,14 @@
 ## finishing setup
 
-To finish your flohmarkt setup you'll need to open its setup URL. 
+To finish your flohmarkt setup you'll need to finish the initialization process at its setup URL.
 
-You can find the setup URL in flohmarkts logfile in /var/log/__APP__ if you look for entries like this:
+See below the configuration tab in the 🔧 wrench section to find the setup URL.
+
+In case of problems with the configuration panel you can find the setup URL in flohmarkts logfile in /var/log/__APP__/app.log if you look for entries like this. To get the entry into the logfile you should at least open your flohmarkt web page once at [https://__DOMAIN__/](https://__DOMAIN__/).
 
 ```
 2024-05-06 16:30:24 Flohmarkt is not initialized yet. Please go to 
-2024-05-06 16:30:24                 https://YOUR_URL/setup/SECRET
+2024-05-06 16:30:24                 https://__DOMAIN__/setup/SECRET
 2024-05-06 16:30:24                 in order to complete the setup process
 ```
-
-(YOUR_URL will reflect the DOMAIN/PATH you chose for installation and SECRET is a random string)
+(SECRET is a random string)

doc/POST_INSTALL_de.md

@@ -0,0 +1,14 @@
+## flohmarkt initialisieren
+
+Um die Installation von flohmarkt abzuschließen muss die Konfiguration unter der Setup URL abgeschlossen werden.
+
+Weiter unten im Abschnitt mit dem 🔧 Schraubenschlüssel findest Du die Konfigurationsseit mit einem Link zu flohmarkts setup Seite.
+
+Bei Problemen mit dem untenstehenden Konfigurationsdialog findest Du die URL auch im flohmarkt Protokoll unter /var/log/__APP__/app.log wie unten im Beispiel gezeigt. Damit der Eintrag im Protokoll erscheint, musst Du einmal Deine flohmarkt Seite unter [https://__DOMAIN__/](https://__DOMAIN__/) öffnen.
+
+```
+2024-05-06 16:30:24 Flohmarkt is not initialized yet. Please go to 
+2024-05-06 16:30:24                 https://__DOMAIN__/setup/SECRET
+2024-05-06 16:30:24                 in order to complete the setup process
+```
+(SECRET ist eine zufällige Zeichenkette)

doc/PRE_UPGRADE.md

@@ -0,0 +1,21 @@
+# new in 0.0~ynh8
+
+## urlwatch
+
+This version configures the regular execution of `urlwatch` to find differences in flohmarkts user database.
+
+The job is called from /etc/cron.hourly/ every hour.
+
+If `urlwatch` finds a difference in the user database stored in CouchDB for a flohmarkt instance it'll send an email to an email address saved as a new setting "admin_mail".
+
+## new setting "admin_mail"
+
+The upgrade script will guess a default:
+
+* it'll try to read the email of the flohmarkt admin from couchdb
+* if it fails for some reason it'll use 'admin@<your main domain>'
+
+You'll get WARNINGs during the upgrade showing you the address being used. Please check the address!
+
+If you do not like the default you can change it on the new config panel you can find in your yunohost admin webgui on the page of the flohmarkt app.
+

manifest.toml

@@ -7,7 +7,7 @@ name = "flohmarkt"
 description.en = "A decentral federated small ads platform"
 description.fr = "Plateforme de petites annonces fédérées décentralisées"
 
-version = "0.0~ynh7"
+version = "0.0~ynh8"
 
 maintainers = ["Chris Vogel"]
 
@@ -97,6 +97,13 @@ ram.runtime = "100M"
     ask.en = "name of your flohmarkt instance"
     ask.de = "Name des Flohmarkts"
 
+    [install.admin_mail]
+    type="email"
+    ask.en = "email address of this flohmarkts admin"
+    help.en = "you'll get notified about events like changes in the user database of flohmarkt"
+    ask.de = "E-Mail Adresse des Administrators dieses flohmarkts"
+    help.de = "an diese Adresse werden Informationen zu z.B. Änderungen an der Benutzerdatenbank geschickt"
+
 [resources]
     # See the packaging documentation for the full set
     # of explanation regarding the behavior and properties for each of those
@@ -155,7 +162,8 @@ ram.runtime = "100M"
     [resources.apt]
     # python dependencies shall be installed in a venv using pip.
 	# moreutils is needed for `ts` used in systemd.service
-    packages = "python3-pip python3-full curl apt-transport-https gnupg moreutils curl"
+    # urlwatch for email notifications
+    packages = "python3-pip python3-full curl apt-transport-https gnupg moreutils curl urlwatch"
     # repo for couchdb - doesn't work, yet
     # extras.couchdb.repo = "deb https://apache.jfrog.io/artifactory/couchdb-deb/ __YNH_DEBIAN_VERSION__ main"
     # extras.couchdb.key = "https://couchdb.apache.org/repo/keys.asc"

scripts/_common.sh

@@ -373,7 +373,7 @@ flohmarkt_ynh_local_curl() {
 # | arg: -m --method:     request method to use: POST (default), PUT, GET, DELETE
 # | arg: -H --header:     add a header to the request (can be used multiple times)
 # | arg: -d --data:       data to be PUT or POSTed. Can be used multiple times.
-# | arg: -s --seperator:  seperator used to concatenate POST/PUT --date or key=value ('none'=no seperator)
+# | arg: -s --seperator:  seperator used to concatenate POST/PUT --data or key=value ('none'=no seperator)
 # | arg:                  (default for POST: '&', default for PUT: ' ')
 # | arg: -u --user:       login username (requires --password)
 # | arg: -p --password:   login password
@@ -642,9 +642,10 @@ flohmarkt_ynh_dump_couchdb() {
 }
 
 flohmarkt_ynh_import_couchdb() {
-  ls -l ../settings/scripts/couchdb-dump/couchdb-dump.sh ${YNH_CWD}/${app}.json
   ../settings/scripts/couchdb-dump/couchdb-dump.sh -r -c -H 127.0.0.1 -d "${app}" \
     -q -u admin -p "${password_couchdb_admin}" -f "${YNH_CWD}/${app}.json"
+  # TODO: This failed silently during tests when there were leftovers from a former
+  # install. Check couchdb-dump.sh restore for error codes and handling.
 }
 
 flohmarkt_ynh_delete_couchdb_user() {
@@ -725,6 +726,15 @@ flohmarkt_ynh_rename_couchdb() {
   flohmarkt_ynh_delete_couchdb_db "$old_name"
 }
 
+flohmarkt_ynh_backup_old_couchdb() {
+  flohmarkt_couchdb_rename_to="${app}_$(date '+%Y-%m-%d_%H-%M-%S_%N')"
+  if flohmarkt_ynh_rename_couchdb "${app}" "${flohmarkt_couchdb_rename_to}"; then
+    ynh_print_warn --message="renamed existing database ${app} to ${flohmarkt_couchdb_rename_to}"
+  else
+    ynh_die --message="could not rename existing couchdb database and cannot overwrite it"
+  fi
+}
+
 flohmarkt_ynh_restore_couchdb() {
   flohmarkt_ynh_import_couchdb
   flohmarkt_ynh_create_couchdb_user
@@ -748,6 +758,36 @@ flohmarkt_ynh_venv_requirements() {
   )
 }
 
+flohmarkt_ynh_urlwatch_cron() {
+    mkdir -m 770 -p "${flohmarkt_install}/urlwatch"
+    chown root:${app} "${flohmarkt_install}/urlwatch"
+    ynh_add_config --template="../conf/urlwatch_config.yaml" \
+        --destination="${flohmarkt_install}/urlwatch/config.yaml"
+    ynh_add_config --template="../conf/urlwatch_urls.yaml" \
+        --destination="${flohmarkt_install}/urlwatch/urls.yaml"
+    ynh_add_config --template="../conf/urlwatch.cron" \
+        --destination="/etc/cron.hourly/${flohmarkt_filename}"
+    chown root:root "/etc/cron.hourly/${flohmarkt_filename}"
+    chmod 755 "/etc/cron.hourly/${flohmarkt_filename}"
+    #  run it once to initialize
+    sudo -u ${app} urlwatch \
+    --config /var/www/${app}/urlwatch/config.yaml \
+    --urls=/var/www/${app}/urlwatch/urls.yaml \
+    --cache /var/www/${app}/urlwatch/cache.file
+}
+
+flohmarkt_initialized() {
+    flohmarkt_ynh_local_curl -n -m GET -u admin -p "$password_couchdb_admin" \
+        -l '"initialized":true' \
+        "http://127.0.0.1:5984/${app}/instance_settings" 
+}
+
+flohmarkt_ynh_get_initialization_key() {
+    flohmarkt_ynh_local_curl -n -m GET -u admin -p "$password_couchdb_admin" \
+        "http://127.0.0.1:5984/${app}/instance_settings" \
+    | jq -r .initialization_key
+}
+
 # move files and directories to their new places
 flohmarkt_ynh_upgrade_path_ynh5() {
   # flohmarkt and couchdb are already stopped in upgrade script

scripts/backup

@@ -11,6 +11,7 @@ ynh_backup --src_path="/etc/fail2ban/filter.d/$app.conf"
 ynh_backup --src_path="/etc/logrotate.d/$app"
 ynh_backup --src_path="/opt/couchdb/etc/local.d/05-flohmarkt.ini"
 ynh_backup --src_path="/etc/systemd/system/$flohmarkt_filename.service"
+ynh_backup --src_path="/etc/cron.hourly/$flohmarkt_filename"
 ynh_backup --src_path="${flohmarkt_log_dir}"
 
 # for the following backups we'll want to stop flohmarkt and couchdb

scripts/config

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+source /etc/yunohost/apps/$app/scripts/_common.sh
+source /usr/share/yunohost/helpers
+
+ynh_abort_if_errors
+
+# my code starts here:
+
+get__flohmarkt_initial_setup_url(){
+    local init_key="$(flohmarkt_ynh_get_initialization_key)"
+    # TODO: wenn leer, dann etwas anderes zurückgeben
+    if ! [[ -v init_key ]] || 
+        [[ "${init_key}" == 'null' ]] || 
+        [[ -z "${init_key}" ]]
+    then
+        if flohmarkt_initialized; then
+            echo '✅ <b>done already</b>'
+        else
+            echo "<b><font color=\"red\">error:</font></b> couldn't find the initialization key"
+        fi
+    else
+        echo -n "<b><a href=\"https://${domain}/setup/${init_key}\" target=\"_blank\">"
+        echo "https://${domain}/setup/${init_key}</a></b>"
+    fi
+}
+
+# https://yunohost.org/en/packaging_config_panels#custom-getter-setter
+# we're supposed to put this here, to fill variables with values from settings
+ynh_app_config_run $1

scripts/install

@@ -121,4 +121,8 @@ ls -l $flohmarkt_logfile /bin/bash /usr/bin/bash || true
 ynh_script_progression --message="Starting flohmarkt..." --weight=10
 flohmarkt_ynh_start_service
 
+# integrate urlwatch
+ynh_script_progression --message="Configure urlwatch and its cron job..." --weight=1
+flohmarkt_ynh_urlwatch_cron
+
 ynh_script_progression --message="Installation of $id completed" --last

scripts/restore

@@ -23,14 +23,10 @@ else
   ynh_script_progression --message="CouchDB configuration file already exists." --weight=1
 fi
 
-# check if couchdb already exists and if so rename it to make room for the restore
-if flohmarkt_ynh_exists_couchdb_db; then
-  flohmarkt_couchdb_rename_to="${app}_$(date '+%Y-%m-%d_%H-%M-%S_%N')"
-  if flohmarkt_ynh_rename_couchdb "${app}" "${flohmarkt_couchdb_rename_to}"; then
-    ynh_print_warn --message="renamed existing database ${app} to ${flohmarkt_couchdb_rename_to}"
-  else
-    ynh_die --message="could not rename existing couchdb database and cannot overwrite it"
-  fi
+# if exists rename existing couchdb
+ynh_script_progression --message="Check if old database exists..." --weight=1
+if flohmarkt_ynh_exists_couchdb_db; then 
+  flohmarkt_ynh_backup_old_couchdb
 fi
 
 # restore couchdb from json
@@ -88,4 +84,8 @@ ynh_systemd_action --service_name=nginx --action=reload
 ynh_script_progression --message="Creating symlinks..." --weight=1
 flohmarkt_ynh_create_symlinks
 
+# restore cronjob file for urlwatch
+ynh_script_progression --message="Restoring cronjob for urlwatch..." --weight=1
+ynh_restore_file --origin_path="/etc/cron.hourly/$flohmarkt_filename"
+
 ynh_script_progression --message="Restoration completed for $app" --last

scripts/upgrade

@@ -88,4 +88,26 @@ ynh_add_fail2ban_config --logpath="$flohmarkt_logfile" --failregex='INFO: +<HOST
 ynh_script_progression --message="Starting flohmarkt..." --weight=3
 flohmarkt_ynh_start_service
 
+# check if admin_mail is already set and if not use a default
+if ! [[ -v admin_mail ]] || [[ -z "${admin_mail}" ]]; then
+  ynh_print_warn --message="new setting admin_mail - trying to set a default"
+  # try to read admin email address from flohmarkt installation
+  admin_mail=$( flohmarkt_ynh_local_curl -n -m POST -u admin -p "$password_couchdb_admin" \
+    -H "Accept: application/json" -H "Content-Type: application/json" -s none \
+    -d '{"selector": { "role": "User", "admin": true }, "fields": ["email"]}' \
+    --location 'http://127.0.0.1:5984/flohmarkt/_find' \
+    | jq -r '.docs[0] .email'
+  )
+  if [[ -z "${admin_mail}" ]] || [[ "${admin_mail}" == 'null' ]]; then
+    # use yunohost admin group
+    admin_mail="admin@$(yunohost domain main-domain | cut -d' ' -f2)"
+  fi
+  ynh_print_warn --message="using ${admin_mail} for new setting admin_mail"
+  ynh_app_setting_set --app=$app --key=admin_mail --value="${admin_mail}"
+fi
+
+# upgrade urlwatch configuration
+ynh_script_progression --message="Upgrade urlwatch configuration..." --weight=1
+flohmarkt_ynh_urlwatch_cron
+
 ynh_script_progression --message="Upgrade of $app completed" --last

tests.toml

@@ -7,6 +7,7 @@ test_format = 1.0
 
     args.password_couchdb_admin = "TestPasswordCouchDBadmin"
     args.password_couchdb_flohmarkt = "TestPasswordCouchDBflohmarkt"
+    args.admin_mail = "admin@example.com"
 
     test_upgrade_from.217fc0e.name = "Upgrade from ynh4"