Commit 2943a36e authored by Anita Graser's avatar Anita Graser
Browse files

initial public commit

parents
__pycache__/
_images/
.mysql-data/
.pytest_cache/
.vscode/
tests/
.git
.cache
.pytest_cache/
.gitignore
gitlab-ci.yml
*.md
node_modules
main.py
devsql.db
\ No newline at end of file
# Environment variable overrides for local development
FLASK_APP=autoapp.py
FLASK_DEBUG=1
FLASK_ENV=development
DATABASE_URL=sqlite:////tmp/dev.db
GUNICORN_WORKERS=1
LOG_LEVEL=debug
SECRET_KEY=not-so-secret
INSTALL_PYTHON_VERSION=3.9.7
INSTALL_NODE_VERSION=16.13
# In production, set to a higher number, like 31556926
SEND_FILE_MAX_AGE_DEFAULT=0
SESSION_COOKIE_SECURE=False
\ No newline at end of file
# Environment variable overrides for local development
FLASK_APP=autoapp.py
FLASK_DEBUG=1
FLASK_ENV=development
DATABASE_URL=sqlite:////tmp/devsql.db
GUNICORN_WORKERS=1
LOG_LEVEL=debug
SECRET_KEY=not-so-secret
# In production, set to a higher number, like 31556926
SEND_FILE_MAX_AGE_DEFAULT=0
OAUTH2_REFRESH_TOKEN_GENERATOR=True
# SQLALCHEMY_TRACK_MODIFICATIONS=False
SESSION_COOKIE_SECURE=True
\ No newline at end of file
module.exports = {
env: {
browser: true,
node: true,
es2021: true
},
extends: [
'standard'
],
parserOptions: {
ecmaVersion: 12,
sourceType: 'module'
},
rules: {
'no-param-reassign': 0,
'import/no-extraneous-dependencies': 0,
'import/prefer-default-export': 0,
'consistent-return': 0,
'no-confusing-arrow': 0,
'no-underscore-dangle': 0,
},
globals: {
__dirname: true,
jQuery: true,
$: true,
},
};
__pycache__/
.devcontainer
.mysql-data
detect/static/build/*
node_modules
*.py[cod]
*$py.class
{
"configurations": [
{
"name": "DETECT local debugger",
"type": "python",
"request": "launch",
"program": "main.py",
"console": "integratedTerminal",
"cwd": "/home/grasera/mal2git/detect-prototype",
"python": "/home/grasera/anaconda3/envs/detect/bin/python",
}
]
}
\ No newline at end of file
This diff is collapsed.
# ================================== BUILDER ===================================
ARG INSTALL_PYTHON_VERSION=${INSTALL_PYTHON_VERSION:-PYTHON_VERSION_NOT_SET}
ARG INSTALL_NODE_VERSION=${INSTALL_NODE_VERSION:-NODE_VERSION_NOT_SET}
FROM node:${INSTALL_NODE_VERSION}-buster-slim AS node
FROM python:${INSTALL_PYTHON_VERSION}-slim-buster AS builder
WORKDIR /app
# Get minimal node executable dependencies
COPY --from=node /usr/local/bin/ /usr/local/bin/
COPY --from=node /usr/lib/ /usr/lib/
# See https://github.com/moby/moby/issues/37965
RUN true
COPY --from=node /usr/local/lib/node_modules /usr/local/lib/node_modules
# python requirements
COPY requirements requirements
RUN pip install --no-cache -r requirements/prod.txt
COPY package.json ./
RUN npm install
COPY webpack.config.js autoapp.py ./
COPY detect detect
COPY assets assets
COPY .env.detect .env
RUN npm run-script build-debug
# ================================= PRODUCTION =================================
FROM python:${INSTALL_PYTHON_VERSION}-slim-buster as production
WORKDIR /app
RUN useradd -m sid
RUN chown -R sid:sid /app
USER sid
ENV PATH="/home/sid/.local/bin:${PATH}"
COPY requirements requirements
RUN pip install --no-cache --user -r requirements/prod.txt
COPY supervisord.conf /etc/supervisor/supervisord.conf
COPY supervisord_programs /etc/supervisor/conf.d
# TODO: only copy required resources below
COPY . .
# copy resources generated by npm run-script build-debug
COPY --from=builder --chown=sid:sid /app/detect/static /app/detect/static
USER root
RUN chmod a+x shell_scripts/supervisord_entrypoint.sh
USER sid
EXPOSE 5000
ENTRYPOINT ["/bin/bash", "shell_scripts/supervisord_entrypoint.sh"]
CMD ["-c", "/etc/supervisor/supervisord.conf"]
# ================================= DEVELOPMENT ================================
FROM builder AS development
RUN pip install --no-cache -r requirements/dev.txt
EXPOSE 2992
EXPOSE 5000
CMD [ "npm", "start" ]
GNU GPLv3
Copyright (c) 2021-2022 AIT DSS DSAI
[[source]]
url = "https://pypi.python.org/simple"
verify_ssl = true
name = "pypi"
[packages]
Flask = "==2.0.1"
Werkzeug = "==2.0.1"
click = ">=8.0"
Flask-SQLAlchemy = "==2.5.1"
SQLAlchemy = "==1.4.23"
psycopg2-binary = "==2.9.1"
Flask-Migrate = "==3.1.0"
email-validator = "==1.1.3"
Flask-WTF = "==0.15.1"
WTForms = "==2.3.3"
gevent = "==21.8.0"
gunicorn = ">=20.1.0"
supervisor = "==4.2.2"
Flask-Static-Digest = "==0.2.1"
Flask-Login = "==0.5.0"
Flask-Bcrypt = "==0.7.1"
Flask-Caching = ">=1.10.1"
Flask-DebugToolbar = "==0.11.0"
environs = "==9.3.3"
[dev-packages]
[requires]
python_version = "3.9"
release: flask db upgrade
web: gunicorn detect.app:create_app\(\) -b 0.0.0.0:$PORT -w 3
[[_TOC_]]
# DETECT
This guide explains how to set up, develop, and deploy the DETECT application "Fake-Shop Explorer".
Fake-Shop Explorer is a prototype of a browser-based gamified crowd-sourcing application (i.e. a game) that collects user/player input on potential fake shops.
Fake-Shop Explorer has been developed and tested on Ubuntu 20.04.4 LTS (native and WSL Windows Subsystem for Linux) for use in Firefox and Chrome.
<img align="right" src="./docs/landing-page.png">
## Overview
Fake-Shop Explorer is implemented as a [Flask](https://flask.palletsprojects.com/en/2.0.x/) application.
It can be run in a [containerised](#docker-setup) setup (using `Docker` and `docker-compose`) or locally without Docker.
In production deployment, Fake-Shop Explorer uses two web services:
1. **Fake shop detector** which has a public API: https://api.fakeshop.at/fake-shop-detector/api/1.2/ui/ and
2. **Fake shop database** https://db.malzwei.at/api/ which requires an access token to use. If you want to use the fake shop database, obtain a token and place it in the path that is defined in `detect/settings.py` for `DEV_DB_API_KEY_LOC`.
As a **fallback** (without token) sample URLs are read from a **local** resource file.
Depending on the game level, players are asked different questions about the websites (potential fake shops). For this purpose, the websites are embedded into the Fake-Shop Explorer interface:
<img align="right" src="./docs/shop-noshop-question.png">
Player inputs are saved to a database (MySQL in production, Sqlite for development).
Since Fake-Shop Explorer can be run locally or using Docker, the following instructions describe both options.
## Local environment setup
To get started, download the sources:
```bash
git clone https://git-service.ait.ac.at/...
```
and change your working directory to the download folder.
### Node and npm
Fake-Shop Explorer development requires **node 16**. (Don't install node 17+ because of https://stackoverflow.com/questions/69719601/getting-error-digital-envelope-routines-reason-unsupported-code-err-oss)
It is recommended to install `nvm`. Based on https://askubuntu.com/a/1009527: To avoid conflicts, remove existing installs:
```bash
sudo apt purge nodejs npm
```
then install nvm LTS and use it
```bash
curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.33.8/install.sh | bash
nvm install --lts
```
Finally, run `npm install` to install the dependencies specified in package.json
### Python environment setup
Fake-Shop Explorer development requires **Python >= 3.9**. A good option to create a dedicated `detect` Python environment for development purposes is to use conda:
```bash
conda create --name detect python=3
```
and - once the `detect` environment is created - activate it and install the development dependencies:
```bash
conda activate detect
pip install -r requirements/dev.txt
```
### Local deployment
After installing node, npm, and Python, the Fake-Shop Explorer can be built and run using:
```bash
npm run-script build # use build-debug for non-minimized javascript
npm start # run the webpack dev server and flask server using concurrently
```
When the app starts, the log will show where it is launching, something like `http://172.30.212.54:5000/`.
The local version of this app uses Sqlite. You can access the database using:
```
sqlite3 /tmp/dev.db
```
## Docker setup
The containerized version of Fake-Shop Explorer consists of three main services: `flask-dev`, `flask-prod`, and `manage`.
Services and images are defined in `docker-compose.yml` resp. `Dockerfile`.
### Steps to start from scratch: development
To get started, clone the repo, build and start the **dev** service:
```bash
git clone https://mal2git.x-t.at/stockingere/detect-prototype.git
cd detect-prototype
docker-compose build --no-cache flask-dev
docker-compose up -d flask-dev
```
Then build the app resources inside the dev container:
```bash
docker exec -it detect-prototype_flask-dev_1 /bin/bash
npm run-script build
exit
docker-compose down
```
*Note: A docker volume `node-modules` is created to store NPM packages and is reused across the dev and prod containers. For the purposes of DB testing with `sqlite`, the file `dev.db` is mounted to all containers. This volume mount should be removed from `docker-compose.yml` if a production DB server is used.*
*Note: The list of `environment:` variables in the `docker-compose.yml` file takes precedence over any variables specified in `.env`.*
#### Cleaning up old containers and database content
In development, it is often easier to start from scratch with an empty database and new containers when the schema or initial table content change. You can remove the old containers and database content using:
```bash
docker image remove detect-development:latest
docker image remove detect-manage:latest
sudo rm -r .mysql-data/
```
### Production deployment
[supervisord](http://supervisord.org/) is used for process management and [gunicorn](https://gunicorn.org/) is used as HTTP server.
Settings for supervisord are defined in `supervisord-conf`, settings for gunicorn are defined in `supervisord_programs/gunicorn.conf`,
Build the production container:
```bash
docker-compose build --no-cache flask-prod
```
Start the container in detached mode:
```bash
docker-compose up -d flask-prod
```
Check if detect-prototype_flask-prod_1 and detect-prototype_db_1 run:
```bash
docker ps
```
To access the logs of a background container (started with -d) use:
```bash
docker logs detect-prototype_flask-dev_1
```
To access the containerized MySQL database, connect to the container:
```bash
docker exec -it detect-prototype_db_1 /bin/bash
```
and run
```bash
mysql -u detect -p detect
```
### Manage
The manage container is used to run commands using the `Flask CLI`:
```bash
docker-compose run --rm manage <<COMMAND>>
```
Therefore, to initialize a database you can run:
```bash
docker-compose run --rm manage db init
docker-compose run --rm manage db migrate
docker-compose run --rm manage db upgrade
```
Similarly, to open the interactive Flask shell, run
```bash
docker-compose run --rm manage db shell
```
## Advanced Developer documentation
### Application settings
Settings can be modified in `detect/settings.py`
### Asset Management
Files placed inside the `assets` directory and its subdirectories
(excluding `js` and `css`) will be copied by webpack's
`file-loader` into the `static/build` directory. In production, the plugin
`Flask-Static-Digest` zips the webpack content and tags them with a MD5 hash.
As a result, you must use the `static_url_for` function when including static content,
as it resolves the correct file name, including the MD5 hash.
For example
```html
<link rel="shortcut icon" href="{{static_url_for('static', filename='build/img/favicon.ico') }}">
```
If all of your static files are managed this way, then their filenames will change whenever their
contents do, and you can ask Flask to tell web browsers that they
should cache all your assets forever by including the following line
in ``.env``:
```text
SEND_FILE_MAX_AGE_DEFAULT=31556926 # one year
```
### DETECT Session
`detect/detect_session.py` wraps the Flask session object and should be used instead of the session object to store user-specific information.
### Concepts
In the game, users travel through space and visit unknown planets.
#### Tasks
Tasks have to be solved to collect fuel for the spaceship, and to map/categorize new planets.
Thus, two task types exist:
* **fuel**/**simple** tasks (level 0): replenish energy for the spaceship by answering a simple yes/no decision "shop or not?"
* **planet** tasks (level 1-7): answer detailed questions about potential web/fake shop sites. Planet tasks become increasingly complicated with level progress.
In the data model, `task` is used synonymously for website url, while `subtask` is synonymous for answering questions about the website url.
### Database schema / Flask models
<img align="right" width="70%" src="./docs/db-schema.png">
`detect/user/models.py` contains the app-specific database tables and view definitions. The full schema is shown in the ER diagram on the right. The tables contain the following information
* `events`: logs user actions, such as visiting planets or gathering fuel
* `options`: answer options for game level questions
* `roles`: user roles, not used yet
* `subtasks`: game level questions
* `tasks`: URLs of the websites / fake-shops
* `user_task_link`: logs user guess regarding shop / no-shop / fake shop classification
* `user_task_subtask_link`: logs user answer to game level questions (e.g. if "Impressum" (site notice) exists or looks plausible)
The views `v_agreement`, `v_accuracy`, and `v_accuracy_monthly` are used to compute how well a user's answers agree with the majority opinion of other users.
`detect/user/models.py` also contains two functions that are called from `app.py` before the database is first queried:
* `create_subtasks_and_options` parses the subtasks (game level questions) defined in `detect/resources/subtasks.json` and inserts them into the the `subtasks` table.
* `create_users` creates some test users for easier testing and should be removed in production.
### Testing
Fake-Shop Explorer comes with an extensive suit of tests.
To run all tests with Docker run:
```bash
docker-compose run --rm manage test
```
Alternatively, without Docker, run:
```bash
flask test
```
#### Running Tests in VSCode
In VSCode, the following launch.json can be used:
```
{
"configurations": [
{
"name": "DETECT local debugger",
"type": "python",
"request": "launch",
"program": "main.py",
"console": "integratedTerminal",
"cwd": "/<your-project-path-here>/detect-prototype",
"python": "/<your-python-path-here>/anaconda3/envs/detect/bin/python",
}
]
}
```
### Linting
The `lint` command will attempt to fix any linting/style errors in the code. If you only want to know if the code will pass CI and do not wish for the linter to make changes, add the `--check` argument.
To run the linter, run:
```bash
docker-compose run --rm manage lint
flask lint # If running locally without Docker
```
## Advanced Devops Documentation
### Database migrations
Whenever a database migration needs to be made.
To generate a new migration script, run:
```bash
docker-compose run --rm manage db migrate
flask db migrate # If running locally without Docker
```
Then apply the migration:
```bash
docker-compose run --rm manage db upgrade
flask db upgrade # If running locally without Docker
```
For a full migration command reference, run `docker-compose run --rm manage db --help`.
If you will deploy your application remotely (e.g on Heroku) you should add the `migrations` folder to version control.
You can do this after `flask db migrate` by running the following commands
```bash
git add migrations/*
git commit -m "Add migrations"
```
Make sure folder `migrations/versions` is not empty.
{
"name": "detect",
"env": {
"SECRET_KEY": {
"description": "SECRET_KEY.",
"generator": "secret"
},
"FLASK_APP": {
"description": "FLASK_APP.",
"value": "autoapp.py"
}
},
"buildpacks": [
{
"url": "heroku/nodejs"
},
{
"url": "heroku/python"
}
],
"addons": [
{
"plan": "heroku-postgresql:hobby-dev",
"options": {
"version": "11"
}
}
]
}
/* z-indices
OVERLAYS are at 7000
elements attracting attention over OVERLAYS are at 8000
NAVBARS (main nav, footer) are are 10000+
*/
.overlay {
background-image: linear-gradient(to bottom, rgba(0, 0, 0, 0.2), rgba(0, 0, 0, 0.8));
z-index: 7000;
}
.over-overlay {
z-index: 8000;
}
@keyframes fadeOut {
0% {
opacity: 1;
}
100% {
opacity: 0;
}
}
/** ALERTS **/
#alertMessages {
position: fixed !important;
animation-name: fadeOut;
-webkit-animation-duration: 15s;
animation-duration: 15s;
-webkit-animation-fill-mode: both;
animation-fill-mode: both;
}
/** Live Dashboard **/