Commit 2b352e0f authored by Steve Weber's avatar Steve Weber
Browse files

more

parent 2fc7a38a
.venv/
.docker/
.private/
settings.py
.local/
.venv/
env
initdb.json*
initdb.sh*
requirements.txt
initdb.json
initdb.sh
settings.py
start.sh
# ============================================================
......
Development workspaces
======================
We provide a few ways to get you up and running.
Use a python virtual environment (venv), Docker or a mix of them.
Python virtual environment
--------------------------
packages
```
# install your local python package or grab it from python.org
command -v apt && sudo apt install python3-venv
command -v yum && sudo yum install python3
# create python virtual environment
python3 -m venv .local/venv
```
config
```
# modify as needed
cp examplefiles/env_local env
cp examplefiles/requirements_local.sh requirements.sh
cp examplefiles/requirements_local.txt requirements.txt
cp examplefiles/settings_local.py settings.py
cp examplefiles/initdb_loaddata.json initdb.json
cp examplefiles/initdb_loaddata.sh initdb.sh
# if you want https
openssl req -x509 -newkey rsa:4096 -nodes -out .local/server.crt -keyout .local/server.key -days 36500 -subj "/C=CA/ST=TEST/L=TEST/O=TEST/CN=localhost"
```
run
```
./init.sh
./start.sh
```
run extra
```
source ./env.sh
python manage.py migrate
python manage.py collectstatic --noinput
python manage.py makemigrations
```
clean
```
rm -rf .local
git clean -fX
```
Docker
------
packages
```
sudo apt install docker-compose
sudo usermod -a -G docker $USER
# reboot or use `su $USER` so `id` showes you in docker
```
config
```
# modify as needed
mkdir .local
cp examplefiles/env_docker .local/env
cp examplefiles/requirements_docker.sh requirements.sh
cp examplefiles/requirements_docker.txt requirements.txt
cp examplefiles/settings_docker.py settings.py
openssl req -x509 -newkey rsa:4096 -nodes -out .local/server.crt -keyout .local/server.key -days 36500 -subj "/C=CA/ST=TEST/L=TEST/O=TEST/CN=localhost"
```
run
```
# (first time will take a while)
docker-compose up
# DONE its running :)
# in another terminal findout what port the webproxy is running on:
docker-compose port nginx 443
## from a webbrowser open https://0.0.0.0:?????/
```
run extra
```
# if you want a shell in a docker
docker-compose run app /bin/bash
# this shell is in a new container; it shares storage volumes but not system files
# from here you can use manage.py to update the database and static files
source ./env.sh
python manage.py migrate
python manage.py collectstatic --noinput
python manage.py makemigrations
```
rebuild
```
docker-compose up --build
```
clean
```
rm -rf .local
git clean -fX
docker-compose down
docker images
docker rmi ???_nginx
docker rmi ???_web
docker rmi ???_db
```
Advanced
--------
prep
```
deploy:
- os: ubuntu 18.04
- database: postgresql
- webproxy: nginx see: dockerfiles/nginx/sites-enabled/app
```
packages
```
# dependencies
command -v apt && sudo apt install python3-venv
command -v yum && sudo yum install python3
```
config
```
# user
groupadd -r webapp
useradd -r -m -d /srv/webapp -g webapp webapp
mkdir -m 0770 /srv/webapp/.ssh
cat > /srv/webapp/.ssh/ed25519 <<KEY
............
KEY
chmod 0700 /srv/webapp/.ssh/ed25519
ssh-keyscan -t ed25519 git.uwaterloo.ca > /srv/webapp/.ssh/known_hosts
chown -R webapp:webapp /srv/webapp/.ssh
# files
sudo -Hu webapp bash << "EOF_user_tasks"
python3 -m venv /srv/webapp/venv
mkdir /srv/webapp/vol
mkdir /srv/webapp/src
git clone <webapp-repo>
cd /srv/webapp/src
cp requirements_example_docker.sh requirements.sh
cp requirements_example_docker.txt requirements.txt
cp ????/settings_example_postgresql.py ????/settings.py
cp initdb_example_local.json initdb.json
#cp initdb_example_sync.sh initdb.sh
EOF_user_tasks
# service file
#/usr/local/lib/systemd/system/webapp.service
cat > /etc/systemd/system/webapp.service <<EOF
[Unit]
Description=webapp
[Service]
Type=simple
SyslogIdentifier=webapp-uwsgi
EnvironmentFile=/etc/default/webapp
ExecStart=/srv/webapp/venv/bin/uwsgi --emperor '/srv/webapp/uwsgi'
ExecStartPost=/bin/sleep 2
Restart=on-failure
RestartSec=15s
[Install]
WantedBy=multi-user.target
EOF
[uwsgi]
plugin = python3
virtualenv = /srv/webapp/venv
uid = webapp
gid = webapp
chdir = {{vars.dir_src}}
module = {{vars.uwsgi_module}}
master = true
processes = 5
socket = :{{vars.uwsgi_port}}
vacuum = true
die-on-term = true
buffer-size = 32768
cat > /etc/default/webapp <<EOF
ENV_VARS=VALUE
EOF
... /srv/webapp/src/<>/settings.py
```
run
```
sudo -Hu webapp /srv/webapp/src/init.sh
systemctl enable webapp
systemctl start webapp
```
update
```
su webapp
git pull
init.sh
```
FAQ
===
How can I set admin:
```
python manage.py createadmin
```
How can I set user as admin:
```
python manage.py shell
...
from django.contrib.auth.models import User
user = User.objects.get(username="<the_user_id>")
user.is_staff = True
user.is_admin = True
user.is_superuser = True
user.save()
```
......@@ -4,8 +4,8 @@ General Information
Manage an overview of projects and general information.
- Website: https://fast.uwaterloo.ca
- Source code: https://git.uwaterloo.ca/fast?
- Issues: https://git.uwaterloo.ca/fast/issues?
- Source: https://git.uwaterloo.ca/fast/project/fast-project-list
- Issues: https://git.uwaterloo.ca/fast/project/fast-project-list/issues
How to embed the overlay; Add the following HTML to your site to embed the FAST contact us overlay.
```
......@@ -15,6 +15,9 @@ How to embed the overlay; Add the following HTML to your site to embed the FAST
</body>
```
For more details see `INSTALL.md`.
Production
==========
......@@ -34,213 +37,3 @@ To test changes in `master` branch on the staging server:
ssh root@fast-t01.private.uwaterloo.ca
salt-call state.apply
```
Development workspaces
======================
We provide two ways to get you up and running.
Use a python virtual environment (venv) or Docker.
Python virtual environment
--------------------------
packages
```
# install your local python package or from python.org
command -v apt && sudo apt install python3-venv
command -v yum && sudo yum install python3
# create python environment
python3 -m venv .local/venv
```
config
```
# modify as needed
cp env_example_local .local/env
cp requirements_example_local.txt requirements.txt
cp projector/settings_example_local.py projector/settings.py
cp initdb_example_local.json initdb.json
# if you want to test https
openssl req -x509 -newkey rsa:4096 -nodes -out .local/server.crt -keyout .local/server.key -days 36500 -subj "/C=CA/ST=TEST/L=TEST/O=TEST/CN=localhost"
```
run
```
source .local/venv/bin/activate
set -a ; source .local/env ; set +a
./init.sh
python ./manage.py runserver
# if you want to test https try gunicorn
#python -m pip install gunicorn
#gunicorn projector.wsgi:application --reload --certfile=.local/server.crt --keyfile=.local/server.key -b 127.0.0.1:8000
```
rebuild
```
rm -rf .local
#! rerun steps [packages, config, run]
```
Docker
------
packages
```
sudo apt install docker-compose
sudo usermod -a -G docker $USER
# reboot or use `su $USER` so `id` showes you in docker
```
config
```
# modify as needed
mkdir .local
cp env_example_docker .local/env
cp requirements_example_docker.sh requirements.sh
cp requirements_example_docker.txt requirements.txt
cp projector/settings_example_postgresql.py projector/settings.py
cp initdb_example_local.json initdb.json
openssl req -x509 -newkey rsa:4096 -nodes -out .local/server.crt -keyout .local/server.key -days 36500 -subj "/C=CA/ST=TEST/L=TEST/O=TEST/CN=localhost"
```
run
```
# (first time will take a while)
docker-compose up
# DONE.
# if you want a shell in a docker
docker-compose run app /bin/bash
# this is a shell in a new app docker; shares storage volumes
# from there you can use manage.py to update the database and files in the shared volumes
python manage.py migrate
python manage.py collectstatic --noinput
python manage.py makemigrations
```
rebuild
```
docker-compose up --build
# if you need a full rebuild...
docker-compose down
docker images
docker rmi ???_nginx
docker rmi ???_web
docker rmi ???_db
rm -rf .local
#! rerun steps [packages, config, run]
```
Production
----------
prep
```
deploy:
- os: ubuntu 18.04
- database: postgresql
- webproxy: nginx
```
packages
```
# user
groupadd -r webapp
useradd -r -m -d /srv/webapp -g webapp webapp
mkdir -m 0770 /srv/webapp/.ssh
cat > /srv/webapp/.ssh/ed25519 <<KEY
............
KEY
chmod 0700 /srv/webapp/.ssh/ed25519
ssh-keyscan -t ed25519 git.uwaterloo.ca > /srv/webapp/.ssh/known_hosts
chown -R webapp:webapp /srv/webapp/.ssh
# dependencies
command -v apt && sudo apt install python3-venv
command -v yum && sudo yum install python3
# files
sudo -Hu webapp bash << "EOF_user_tasks"
python3 -m venv /srv/webapp/venv
mkdir /srv/webapp/vol
mkdir /srv/webapp/src
git clone <webapp-repo>
cd /srv/webapp/src
cp requirements_example_docker.sh requirements.sh
cp requirements_example_docker.txt requirements.txt
cp ????/settings_example_postgresql.py ????/settings.py
cp initdb_example_local.json initdb.json
#cp initdb_example_sync.sh initdb.sh
EOF_user_tasks
# service file
#/usr/local/lib/systemd/system/webapp.service
cat > /etc/systemd/system/webapp.service <<EOF
[Unit]
Description=webapp
[Service]
Type=simple
SyslogIdentifier=webapp-uwsgi
EnvironmentFile=/etc/default/webapp
ExecStart=/srv/webapp/venv/bin/uwsgi --emperor '/srv/webapp/uwsgi'
ExecStartPost=/bin/sleep 2
Restart=on-failure
RestartSec=15s
[Install]
WantedBy=multi-user.target
EOF
```
config
```
[uwsgi]
plugin = python3
virtualenv = /srv/webapp/venv
uid = webapp
gid = webapp
chdir = {{vars.dir_src}}
module = {{vars.uwsgi_module}}
master = true
processes = 5
socket = :{{vars.uwsgi_port}}
vacuum = true
die-on-term = true
buffer-size = 32768
cat > /etc/default/webapp <<EOF
ENV_VARS=VALUE
EOF
... /srv/webapp/src/<>/settings.py
```
run
```
sudo -Hu webapp /srv/webapp/src/init.sh
systemctl enable webapp
systemctl start webapp
```
FAQ
===
How can I set admin:
```
python manage.py createadmin
```
How can I set my user as admin:
```
python manage.py shell
...
from django.contrib.auth.models import User
user = User.objects.get(username="your_user_id")
user.is_staff = True
user.is_admin = True
user.is_superuser = True
user.save()
```
version: '2'
services:
db:
# image: postgres
build: dockerfiles/db
# image: postgres
env_file:
- dockerfiles/db/env
- .local/env
ports:
- "9000:5432"
#ports:
# - "5432"
volumes:
- postgres-db-volume:/data/postgres
- db-postgres-volume:/data/postgres
app:
build: .
#build: dockerfiles/app
env_file:
- dockerfiles/app/env
- .local/env
ports:
- "9001:8000"
#ports:
# - "8000"
volumes:
- .local:/srv/app/.local
- .local/vol/static:/srv/app/vol/static
......@@ -26,15 +24,14 @@ services:
- .:/srv/app/src
depends_on:
- db
nginx:
build: dockerfiles/nginx
env_file:
- dockerfiles/nginx/env
- .local/env
ports:
- "9080:80"
- "9443:443"
- "0:80"
- "0:443"
volumes_from:
- app
restart: always
......@@ -42,6 +39,7 @@ services:
- app:app
depends_on:
- app
#command: /bin/bash -c "envsubst < /etc/nginx/conf.d/mysite.template > /etc/nginx/conf.d/default.conf && exec nginx -g 'daemon off;'"
volumes:
postgres-db-volume:
db-postgres-volume:
FROM tutum/nginx
FROM nginx
RUN rm /etc/nginx/sites-enabled/default
COPY sites-enabled/ /etc/nginx/sites-enabled
COPY app.conf /etc/nginx/sites-enabled/app.conf
# Redirect all non-encrypted to encrypted
server {
server_name app.localhost localhost;
listen 80;
server_name fpl-app.localhost localhost;
return 301 https://$host$request_uri;
}
server {
listen 443 ssl;
server_name app.localhost localhost;
server_name fpl-app.localhost localhost;
charset utf-8;
ssl on;
......
NGINX_SSL_CERTIFICATE=/srv/app/.local/server.crt
NGINX_SSL_CERTIFICATE_KEY=/srv/app/.local/server.key
#!/usr/bin/env bash
# To activate the env run this helper using:
# source ./env.sh
set -a ; source ./env ; set +a
test "$VIRTUAL_ENV" = "" || $VIRTUAL_ENV/bin/activate
DEBUG=1
SECRET_KEY=xxSOME_RAMDOM_TEXTxxx
POSTGRES_USER=user
POSTGRES_PASSWORD=xxSOME_RAMDOM_TEXTxxx
EMAIL_HOST_USER=email
EMAIL_HOST_PASSWORD=xxxxxxxx
PYTHON_BIN=python3
DJANGO_WSGI_MODULE=projector.wsgi
DJANGO_SETTINGS_MODULE=settings
DEBUG=1
SECRET_KEY=xxSOME_RAMDOM_TEXTxxx
POSTGRES_USER=appuser
......
PYTHON_BIN=.venv/bin/python3
PYTHON_BIN=python3
VIRTUAL_ENV=.local/venv
DJANGO_SETTINGS_MODULE=settings
DEBUG=1
SECRET_KEY=xxSOME_RAMDOM_TEXTxxx
EMAIL_HOST_USER=email
EMAIL_HOST_PASSWORD=xxxxxxxx
WEBSERVER_CERTFILE=x
WEBSERVER_KEYFILE=x
#!/usr/bin/env bash
$PYTHON_BIN ./manage.py loaddata initdb.json
rm initdb.json
#!/bin/bash
# load env vars
# database to sync from
DB_SOURCE_USER
DB_SOURCE_PASSWORD
DB_SOURCE_HOST
DB_SOURCE_NAME
# database to sync to
DB_USER
DB_PASSWORD
DB_HOST
DB_NAME
command -v apt && apt -y install postgresql-client
# Drop all objects in database without dropping the database it's self
echo ""
echo "**** WIPE DATABASE $DB_HOST/$DB_NAME ****"
PGPASSWORD={{vars.settings.databases.default.PASSWORD}} psql \
--host={{vars.settings.databases.default.HOST}} \
--username={{vars.settings.databases.default.USER}} \
--dbname={{vars.settings.databases.default.NAME}} << "EOF"
DO $$
DECLARE
r RECORD;
BEGIN
-- triggers
FOR r IN (SELECT pns.nspname, pc.relname, pt.tgname
FROM pg_trigger pt, pg_class pc, pg_namespace pns
WHERE pns.oid=pc.relnamespace AND pc.oid=pt.tgrelid
AND pns.nspname NOT IN ('information_schema', 'pg_catalog', 'pg_toast')
AND pt.tgisinternal=false
) LOOP
EXECUTE format('DROP TRIGGER %I ON %I.%I;',
r.tgname, r.nspname, r.relname);
END LOOP;
-- constraints #1: foreign key