initial public commit

.dockerignore

+__pycache__/
+_images/
+.mysql-data/
+.pytest_cache/
+.vscode/
+tests/
+.git
+.cache
+.pytest_cache/
+.gitignore
+gitlab-ci.yml
+*.md
+node_modules
+main.py
+devsql.db
\ No newline at end of file

.env

+# Environment variable overrides for local development
+FLASK_APP=autoapp.py
+FLASK_DEBUG=1
+FLASK_ENV=development
+DATABASE_URL=sqlite:////tmp/dev.db
+GUNICORN_WORKERS=1
+LOG_LEVEL=debug
+SECRET_KEY=not-so-secret
+INSTALL_PYTHON_VERSION=3.9.7
+INSTALL_NODE_VERSION=16.13
+# In production, set to a higher number, like 31556926
+SEND_FILE_MAX_AGE_DEFAULT=0
+SESSION_COOKIE_SECURE=False
\ No newline at end of file

.env.detect

+# Environment variable overrides for local development
+FLASK_APP=autoapp.py
+FLASK_DEBUG=1
+FLASK_ENV=development
+DATABASE_URL=sqlite:////tmp/devsql.db
+GUNICORN_WORKERS=1
+LOG_LEVEL=debug
+SECRET_KEY=not-so-secret
+
+# In production, set to a higher number, like 31556926
+SEND_FILE_MAX_AGE_DEFAULT=0
+OAUTH2_REFRESH_TOKEN_GENERATOR=True
+# SQLALCHEMY_TRACK_MODIFICATIONS=False
+SESSION_COOKIE_SECURE=True
\ No newline at end of file

.eslintrc.js

+module.exports = {
+  env: {
+    browser: true,
+    node: true,
+    es2021: true
+  },
+  extends: [
+    'standard'
+  ],
+  parserOptions: {
+    ecmaVersion: 12,
+    sourceType: 'module'
+  },
+  rules: {
+    'no-param-reassign': 0,
+    'import/no-extraneous-dependencies': 0,
+    'import/prefer-default-export': 0,
+    'consistent-return': 0,
+    'no-confusing-arrow': 0,
+    'no-underscore-dangle': 0,
+  },
+  globals: {
+    __dirname: true,
+    jQuery: true,
+    $: true,
+  },
+};

.gitignore

+__pycache__/
+.devcontainer
+.mysql-data
+detect/static/build/*
+node_modules
+
+*.py[cod]
+*$py.class

.vscode/launch.json

+{
+    "configurations": [
+        {
+            "name": "DETECT local debugger",
+            "type": "python",
+            "request": "launch",
+            "program": "main.py",
+            "console": "integratedTerminal",            
+            "cwd": "/home/grasera/mal2git/detect-prototype",
+            "python": "/home/grasera/anaconda3/envs/detect/bin/python",
+        }
+    ]
+}
\ No newline at end of file

Dockerfile

+# ================================== BUILDER ===================================
+
+ARG INSTALL_PYTHON_VERSION=${INSTALL_PYTHON_VERSION:-PYTHON_VERSION_NOT_SET}
+ARG INSTALL_NODE_VERSION=${INSTALL_NODE_VERSION:-NODE_VERSION_NOT_SET}
+
+FROM node:${INSTALL_NODE_VERSION}-buster-slim AS node
+FROM python:${INSTALL_PYTHON_VERSION}-slim-buster AS builder
+
+WORKDIR /app
+
+# Get minimal node executable dependencies
+COPY --from=node /usr/local/bin/ /usr/local/bin/
+COPY --from=node /usr/lib/ /usr/lib/
+# See https://github.com/moby/moby/issues/37965
+RUN true
+COPY --from=node /usr/local/lib/node_modules /usr/local/lib/node_modules
+# python requirements
+COPY requirements requirements
+RUN pip install --no-cache -r requirements/prod.txt
+
+COPY package.json ./
+RUN npm install
+
+COPY webpack.config.js autoapp.py ./
+COPY detect detect
+COPY assets assets
+COPY .env.detect .env
+
+
+RUN npm run-script build-debug
+
+# ================================= PRODUCTION =================================
+FROM python:${INSTALL_PYTHON_VERSION}-slim-buster as production
+
+WORKDIR /app
+
+RUN useradd -m sid
+RUN chown -R sid:sid /app
+USER sid
+ENV PATH="/home/sid/.local/bin:${PATH}"
+
+COPY requirements requirements
+RUN pip install --no-cache --user -r requirements/prod.txt
+
+COPY supervisord.conf /etc/supervisor/supervisord.conf
+COPY supervisord_programs /etc/supervisor/conf.d
+# TODO: only copy required resources below
+COPY . .
+# copy resources generated by npm run-script build-debug
+COPY --from=builder --chown=sid:sid /app/detect/static /app/detect/static
+
+USER root
+RUN chmod a+x shell_scripts/supervisord_entrypoint.sh
+USER sid
+
+EXPOSE 5000
+ENTRYPOINT ["/bin/bash", "shell_scripts/supervisord_entrypoint.sh"]
+CMD ["-c", "/etc/supervisor/supervisord.conf"]
+
+
+# ================================= DEVELOPMENT ================================
+FROM builder AS development
+RUN pip install --no-cache -r requirements/dev.txt
+EXPOSE 2992
+EXPOSE 5000
+CMD [ "npm", "start" ]

LICENSE

+GNU GPLv3
+
+Copyright (c) 2021-2022 AIT DSS DSAI 

Pipfile

+[[source]]
+url = "https://pypi.python.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+Flask = "==2.0.1"
+Werkzeug = "==2.0.1"
+click = ">=8.0"
+Flask-SQLAlchemy = "==2.5.1"
+SQLAlchemy = "==1.4.23"
+psycopg2-binary = "==2.9.1"
+Flask-Migrate = "==3.1.0"
+email-validator = "==1.1.3"
+Flask-WTF = "==0.15.1"
+WTForms = "==2.3.3"
+gevent = "==21.8.0"
+gunicorn = ">=20.1.0"
+supervisor = "==4.2.2"
+Flask-Static-Digest = "==0.2.1"
+Flask-Login = "==0.5.0"
+Flask-Bcrypt = "==0.7.1"
+Flask-Caching = ">=1.10.1"
+Flask-DebugToolbar = "==0.11.0"
+environs = "==9.3.3"
+
+[dev-packages]
+
+[requires]
+python_version = "3.9"

Procfile

+release: flask db upgrade
+web: gunicorn detect.app:create_app\(\) -b 0.0.0.0:$PORT -w 3

README.md

+[[_TOC_]]
+
+
+# DETECT
+
+This guide explains how to set up, develop, and deploy the DETECT application "Fake-Shop Explorer". 
+
+Fake-Shop Explorer is a prototype of a browser-based gamified crowd-sourcing application (i.e. a game) that collects user/player input on potential fake shops. 
+
+
+
+Fake-Shop Explorer has been developed and tested on Ubuntu 20.04.4 LTS (native and WSL Windows Subsystem for Linux) for use in Firefox and Chrome.
+
+
+<img align="right" src="./docs/landing-page.png">
+
+
+
+
+## Overview
+
+Fake-Shop Explorer is implemented as a [Flask](https://flask.palletsprojects.com/en/2.0.x/) application. 
+It can be run in a [containerised](#docker-setup) setup (using `Docker` and `docker-compose`) or locally without Docker.
+
+In production deployment, Fake-Shop Explorer uses two web services: 
+
+1. **Fake shop detector** which has a public API: https://api.fakeshop.at/fake-shop-detector/api/1.2/ui/ and
+2. **Fake shop database** https://db.malzwei.at/api/ which requires an access token to use. If you want to use the fake shop database, obtain a token and place it in the path that is defined in `detect/settings.py` for `DEV_DB_API_KEY_LOC`. 
+As a **fallback** (without token) sample URLs are read from a **local** resource file. 
+
+
+Depending on the game level, players are asked different questions about the websites (potential fake shops). For this purpose, the websites are embedded into the Fake-Shop Explorer interface:
+
+<img align="right" src="./docs/shop-noshop-question.png">
+
+Player inputs are saved to a database (MySQL in production, Sqlite for development).
+
+Since Fake-Shop Explorer can be run locally or using Docker, the following instructions describe both options. 
+
+
+## Local environment setup
+
+To get started, download the sources:
+
+```bash
+git clone https://git-service.ait.ac.at/...
+```
+
+and change your working directory to the download folder. 
+
+
+### Node and npm
+
+Fake-Shop Explorer development requires **node 16**. (Don't install node 17+ because of https://stackoverflow.com/questions/69719601/getting-error-digital-envelope-routines-reason-unsupported-code-err-oss) 
+
+It is recommended to install `nvm`. Based on https://askubuntu.com/a/1009527: To avoid conflicts, remove existing installs:
+
+```bash
+sudo apt purge nodejs npm
+```
+
+then install nvm LTS and use it
+
+```bash
+curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.33.8/install.sh | bash
+nvm install --lts
+```
+
+Finally, run `npm install` to install the dependencies specified in package.json
+
+
+### Python environment setup
+
+Fake-Shop Explorer development requires **Python >= 3.9**. A good option to create a dedicated `detect` Python environment for development purposes is to use conda:
+
+```bash
+conda create --name detect python=3
+```
+
+and - once the `detect` environment is created - activate it and install the development dependencies: 
+
+```bash
+conda activate detect
+pip install -r requirements/dev.txt
+```
+
+
+### Local deployment 
+
+After installing node, npm, and Python, the Fake-Shop Explorer can be built and run using: 
+
+```bash
+npm run-script build # use build-debug for non-minimized javascript
+npm start  # run the webpack dev server and flask server using concurrently
+```
+
+When the app starts, the log will show where it is launching, something like `http://172.30.212.54:5000/`. 
+
+The local version of this app uses Sqlite. You can access the database using:
+
+```
+sqlite3 /tmp/dev.db
+```
+
+
+## Docker setup
+
+The containerized version of Fake-Shop Explorer consists of three main services: `flask-dev`, `flask-prod`, and `manage`.
+
+Services and images are defined in `docker-compose.yml` resp. `Dockerfile`.
+
+
+### Steps to start from scratch: development
+
+To get started, clone the repo, build and start the **dev** service:
+
+```bash
+git clone https://mal2git.x-t.at/stockingere/detect-prototype.git
+cd detect-prototype
+
+docker-compose build --no-cache flask-dev
+docker-compose up -d flask-dev
+```
+
+Then build the app resources inside the dev container:
+
+```bash
+docker exec -it detect-prototype_flask-dev_1 /bin/bash
+npm run-script build
+exit
+
+docker-compose down
+```
+
+*Note: A docker volume `node-modules` is created to store NPM packages and is reused across the dev and prod containers. For the purposes of DB testing with `sqlite`, the file `dev.db` is mounted to all containers. This volume mount should be removed from `docker-compose.yml` if a production DB server is used.*
+
+*Note: The list of `environment:` variables in the `docker-compose.yml` file takes precedence over any variables specified in `.env`.*
+
+#### Cleaning up old containers and database content
+
+In development, it is often easier to start from scratch with an empty database and new containers when the schema or initial table content change. You can remove the old containers and database content using:  
+
+```bash
+docker image remove detect-development:latest
+docker image remove detect-manage:latest
+sudo rm -r .mysql-data/
+```
+
+
+### Production deployment
+
+[supervisord](http://supervisord.org/) is used for process management and [gunicorn](https://gunicorn.org/) is used as HTTP server. 
+Settings for supervisord are defined in `supervisord-conf`, settings for gunicorn are defined in `supervisord_programs/gunicorn.conf`,  
+
+Build the production container:
+
+```bash
+docker-compose build --no-cache flask-prod
+```
+
+Start the container in detached mode:
+
+```bash
+docker-compose up -d flask-prod
+```
+
+Check if detect-prototype_flask-prod_1 and detect-prototype_db_1 run: 
+
+```bash
+docker ps
+```
+
+To access the logs of a background container (started with -d) use:
+
+```bash
+docker logs detect-prototype_flask-dev_1
+```
+
+To access the containerized MySQL database, connect to the container: 
+
+```bash
+docker exec -it detect-prototype_db_1 /bin/bash
+```
+
+and run
+
+```bash
+mysql -u detect -p detect
+```
+
+
+### Manage
+
+The manage container is used to run commands using the `Flask CLI`:
+
+```bash
+docker-compose run --rm manage <<COMMAND>>
+```
+
+Therefore, to initialize a database you can run:
+
+```bash
+docker-compose run --rm manage db init
+docker-compose run --rm manage db migrate
+docker-compose run --rm manage db upgrade
+```
+
+Similarly, to open the interactive Flask shell, run
+
+```bash
+docker-compose run --rm manage db shell
+```
+
+
+## Advanced Developer documentation
+
+### Application settings
+
+Settings can be modified in `detect/settings.py`
+
+### Asset Management
+
+Files placed inside the `assets` directory and its subdirectories
+(excluding `js` and `css`) will be copied by webpack's
+`file-loader` into the `static/build` directory. In production, the plugin
+`Flask-Static-Digest` zips the webpack content and tags them with a MD5 hash.
+As a result, you must use the `static_url_for` function when including static content,
+as it resolves the correct file name, including the MD5 hash.
+For example
+
+```html
+<link rel="shortcut icon" href="{{static_url_for('static', filename='build/img/favicon.ico') }}">
+```
+
+If all of your static files are managed this way, then their filenames will change whenever their
+contents do, and you can ask Flask to tell web browsers that they
+should cache all your assets forever by including the following line
+in ``.env``:
+
+```text
+SEND_FILE_MAX_AGE_DEFAULT=31556926  # one year
+```
+
+
+### DETECT Session
+
+`detect/detect_session.py` wraps the Flask session object and should be used instead of the session object to store user-specific information.
+
+
+### Concepts
+
+In the game, users travel through space and visit unknown planets. 
+
+#### Tasks
+Tasks have to be solved to collect fuel for the spaceship, and to map/categorize new planets.
+Thus, two task types exist:
+
+  * **fuel**/**simple** tasks (level 0): replenish energy for the spaceship by answering a simple yes/no decision "shop or not?"
+  * **planet** tasks (level 1-7): answer detailed questions about potential web/fake shop sites. Planet tasks become increasingly complicated with level progress.  
+
+
+In the data model, `task` is used synonymously for website url, while `subtask` is synonymous for answering questions about the website  url.
+
+
+
+### Database schema / Flask models
+
+<img align="right" width="70%" src="./docs/db-schema.png">
+
+`detect/user/models.py` contains the app-specific database tables and view definitions. The full schema is shown in the ER diagram on the right. The tables contain the following information
+
+* `events`: logs user actions, such as visiting planets or gathering fuel
+* `options`: answer options for game level questions
+* `roles`: user roles, not used yet
+* `subtasks`: game level questions
+* `tasks`: URLs of the websites / fake-shops
+* `user_task_link`: logs user guess regarding shop / no-shop / fake shop classification
+* `user_task_subtask_link`: logs user answer to game level questions (e.g. if "Impressum" (site notice) exists or looks plausible)
+
+The views `v_agreement`, `v_accuracy`, and `v_accuracy_monthly` are used to compute how well a user's answers agree with the majority opinion of other users. 
+
+`detect/user/models.py` also contains two functions that are called from `app.py` before the database is first queried:
+
+* `create_subtasks_and_options` parses the subtasks (game level questions) defined in `detect/resources/subtasks.json` and inserts them into the the `subtasks` table. 
+* `create_users` creates some test users for easier testing and should be removed in production.
+
+
+### Testing
+
+Fake-Shop Explorer comes with an extensive suit of tests.
+
+To run all tests with Docker run:
+
+```bash
+docker-compose run --rm manage test
+```
+
+Alternatively, without Docker, run: 
+
+```bash
+flask test  
+```
+
+
+#### Running Tests in VSCode
+
+In VSCode, the following launch.json can be used:
+
+```
+{
+    "configurations": [
+        {
+            "name": "DETECT local debugger",
+            "type": "python",
+            "request": "launch",
+            "program": "main.py",
+            "console": "integratedTerminal",            
+            "cwd": "/<your-project-path-here>/detect-prototype",
+            "python": "/<your-python-path-here>/anaconda3/envs/detect/bin/python",
+        }
+    ]
+}
+```
+
+
+### Linting
+
+The `lint` command will attempt to fix any linting/style errors in the code. If you only want to know if the code will pass CI and do not wish for the linter to make changes, add the `--check` argument.
+
+To run the linter, run:
+
+```bash
+docker-compose run --rm manage lint
+flask lint # If running locally without Docker
+```
+
+
+
+## Advanced Devops Documentation
+
+### Database migrations
+
+Whenever a database migration needs to be made. 
+To generate a new migration script, run: 
+
+```bash
+docker-compose run --rm manage db migrate
+flask db migrate # If running locally without Docker
+```
+
+Then apply the migration:
+
+```bash
+docker-compose run --rm manage db upgrade
+flask db upgrade # If running locally without Docker
+```
+
+For a full migration command reference, run `docker-compose run --rm manage db --help`.
+
+If you will deploy your application remotely (e.g on Heroku) you should add the `migrations` folder to version control.
+You can do this after `flask db migrate` by running the following commands
+
+```bash
+git add migrations/*
+git commit -m "Add migrations"
+```
+
+Make sure folder `migrations/versions` is not empty.

app.json

+{
+   "name": "detect",
+   "env": {
+      "SECRET_KEY": {
+         "description": "SECRET_KEY.",
+         "generator": "secret"
+      },
+      "FLASK_APP": {
+         "description": "FLASK_APP.",
+         "value": "autoapp.py"
+      }
+   },
+   "buildpacks": [
+      {
+         "url": "heroku/nodejs"
+      },
+      {
+         "url": "heroku/python"
+      }
+   ],
+   "addons": [
+      {
+         "plan": "heroku-postgresql:hobby-dev",
+         "options": {
+            "version": "11"
+         }
+      }
+   ]
+}

assets/css/_layout.scss

+/* z-indices
+
+OVERLAYS are at 7000
+elements attracting attention over OVERLAYS are at 8000
+
+NAVBARS (main nav, footer) are are 10000+
+ */
+
+.overlay {
+    background-image: linear-gradient(to bottom, rgba(0, 0, 0, 0.2), rgba(0, 0, 0, 0.8));
+    z-index: 7000;
+}
+
+.over-overlay {
+    z-index: 8000;
+}
+
+
+
+@keyframes fadeOut {
+    0% {
+        opacity: 1;
+    }
+
+    100% {
+        opacity: 0;
+    }
+}
+
+/** ALERTS **/
+
+#alertMessages {
+    position: fixed !important;
+    animation-name: fadeOut;
+    -webkit-animation-duration: 15s;
+    animation-duration: 15s;
+    -webkit-animation-fill-mode: both;
+    animation-fill-mode: both;
+}
+
+/** Live Dashboard **/