Added hooks and finalized tracked document API

hooks/README.md

+# FAST Hooks
+
+Mirko got it into his head this would be a good idea, and thus it now exists. FAST now supports very basic hooks which trigger various actions. At the moment this is only being used for sending emails on failed cron jobs.
+
+A python3 script is provided in this directory `fast_hook`, which can be run as follows:
+
+```
+fast_hook -k <project key> -e fast/endpoint -v key value -v key2 value2 '<command>'  
+```
+
+The project keys are available in the admin panel for a given fast projct, and hook endpoints are `<project_slug>/<endpoint>`.
+
+The script, by default, will send `stderr` to the endpoint. Any `-v` flags will be included in the JSON data posted to the hook. If you'd like to include `stdout` in the post add an `-o` flag to the command. See `fast_hook --help` for more info.
+
+## Setting Up Hooks
+
+Right now the hooks are super limited in what they can do and who can set them up. They only support email, and configuration needs to be done via manual JSON.
+
+> This workflow may be improved in the future if the feature is used my multiple parties. Notably adding a UI for editing the stupid config.
+
+Note: **you can only edit hooks for projects you've been set as a contact for**. This was mostly done to reduce the amount of noise in the UI.
+
+Here's an example hook setup:
+
+```
+project:      fast
+endpoint:     notify
+type:         email
+configuration:
+{
+    "to": "mvucicev@uwaterloo.ca",
+    "body": "The command {{post_data.command|safe}} failed with the following output: {{post_data.stderr|safe}}",
+    "subject": "Alert: [{{title}}] process failed." 
+}
+```
+
+> body and subject can be left blank and some defaults will be used.
+
+Then hit the hook with the following:
+```
+fast_hook -v title 'Test Command' -k <key> -e fast/notify 'ls /zzz'
+```
+
+The command should fail, and the email 
+
+## I don't want to use your stupid wrapper script
+That's fine! You can just use curl or something:
+
+```
+curl -X POST  https://fast.uwaterloo.ca/hooks/<project>/<endpoint> -H 'key:<key>'  
+```
+
+## Why didn't you just write a wrapper script that sent you emails or used CRON's emailing system?
+Hopefully this can be extended -- it should allow for way more options 
+
+## What if I want a hook endpoint not attached to a specific project?
+I recommend creating a new "invisible" project -- create a new project and simply set "Show on FAST Website" to false.
+
+

hooks/admin.py

+from django.contrib import admin
+from .models import Hook, HookAccessLog
+from projector.admin import OwnProjectFilter, LimitProjectDropdownMixin
+from django.db.models import F
+# Register your models here.
+
+class HookFilter(admin.SimpleListFilter):
+    title = "Hook"
+    parameter_name = 'hook'
+
+    def lookups(self, request, model_admin):
+        return [(x[0], "{}/{}".format(x[1], x[2]))
+            for x in model_admin.get_queryset(request)
+            .filter(hook__project__contact__user=request.user)
+            .values_list('hook__pk', 'hook__project__slug', 'hook__endpoint')
+        ]
+    
+    def queryset(self, request, queryset):
+        if self.value():
+            return queryset.filter(hook=self.value())
+
+class HookProjectFilter(admin.SimpleListFilter):
+    title = "Project"
+    parameter_name = 'project'
+
+    def lookups(self, request, model_admin):
+        return model_admin.get_queryset(request)\
+            .filter(hook__project__contact__user=request.user)\
+            .values_list('hook__project__pk', 'hook__project__title')
+    
+    def queryset(self, request, queryset):
+        if self.value():
+            return queryset.filter(hook__project=self.value())
+
+@admin.register(Hook)
+class HookAdmin(LimitProjectDropdownMixin, admin.ModelAdmin):
+    list_filter = [OwnProjectFilter]
+    list_display = ['endpoint', 'project_title']
+
+
+    def project_title(self, obj):
+        return obj.project.title
+
+    project_title.admin_order_field = 'project__title'
+
+    def get_queryset(self, request):
+        qs = super().get_queryset(request)
+        return qs.filter(project__contact__user=request.user).select_related('project')
+
+@admin.register(HookAccessLog)
+class HookAccessLogAdmin(admin.ModelAdmin):
+    list_filter = [HookProjectFilter, HookFilter]
+    list_display = ['timestamp', 'status', 'project', 'endpoint']
+
+
+    def has_change_permission(*args, **kwargs):
+        return False
+
+    def get_queryset(self, request):
+        qs = super().get_queryset(request)
+        return qs.filter(hook__project__contact__user=request.user)\
+            .annotate(project=F('hook__project__title'))\
+            .annotate(endpoint=F('hook__endpoint'))
+    
+    def project(self, obj):
+        return obj.project
+    
+    def endpoint(self, obj):
+        return obj.endpoint

hooks/apps.py

+from django.apps import AppConfig
+
+
+class HooksConfig(AppConfig):
+    name = 'hooks'

hooks/fast_hook

+#!/usr/bin/env python3
+
+import asyncio
+import argparse
+import json
+import re
+import urllib.request, urllib.error
+
+parser = argparse.ArgumentParser(description='Wraps a command, sends output to a fast hook')
+parser.add_argument(
+    '--key', '-k', required=True,
+    help='Your project key (found in fast admin panel for project)'
+)
+parser.add_argument(
+    '--add-value', '-v', nargs=2, action="append",
+    help="Add extra values to your output json -- eg. -v title Hello -v type string"
+)
+parser.add_argument(
+    '--endpoint', '-e', required=True,
+    help="The hook endpoint -- project_slug/endpoint"
+)
+parser.add_argument(
+    '--include-stdout', '-o', default=False, action='store_true',
+    help="The default setup won't include stdout in the POST data"
+)
+parser.add_argument(
+    '--mode', '-m', default='err',
+    help=(
+        "What mode to run in. Options are err, out, or always.\n"
+        "- err: Only trigger on non-null stderr\n"
+        "- out: Trigger on non-null stdout (includes stdout automatically)\n"
+        "- always: Always trigger even if no output"
+    )
+)
+parser.add_argument(
+    '--hooks-url', '-u', default='https://fast.uwaterloo.ca/hooks',
+    help="Defaults to https://fast.uwaterloo.ca/hooks"
+)
+parser.add_argument('command', nargs='*')
+args = parser.parse_args()
+
+# Clean up and validate args
+if not re.match(r'^[a-z0-9-_]+\/[a-z0-9-_]+$', args.endpoint):
+    print("Invalid endpoint. Must be in the form 'project-slug/endpoint'")
+    exit(1)
+if not len(args.command):
+    print("No command found.")
+    exit(1)
+
+command = " ".join(args.command)
+
+
+def trigger_hook(data):
+    url = f'{args.hooks_url}/{args.endpoint}'
+    print(f"Triggering hook @ {url}")
+    req = urllib.request.Request(
+        url,
+        data=json.dumps(data).encode('utf-8'),
+        headers={
+            'content-type': 'application/json',
+            'key': args.key
+        }
+    )
+    try:
+        urllib.request.urlopen(req)
+    except urllib.error.HTTPError as e:
+        print(f'Failed to trigger: {e}')
+        print(e.read().decode())
+
+async def run(cmd):
+    output_data = {'status': 'OK'}
+    if args.add_value:
+        for r in args.add_value:
+            output_data[r[0]] = r[1]
+    proc = await asyncio.create_subprocess_shell(
+        cmd,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE
+    )
+
+    stdout, stderr = await proc.communicate()
+
+    print(f'-{cmd!r} exited with {proc.returncode}-')
+    if stdout:
+        out = stdout.decode()
+        print(out)
+        if args.include_stdout or args.mode == 'out':
+            output_data['stdout'] = out
+    if stderr:
+        output_data['status'] = 'ERR'
+        err = stderr.decode()
+        print(f'[stderr]\n{err}')
+        output_data['stderr'] = err
+    
+    # some quick checks
+    has_error = output_data.get('stderr', None) is not None
+    has_output = output_data.get('stdout', None) is not None
+
+    if (
+        args.mode == 'err' and has_error
+        or args.mode == 'out' and has_output
+        or args.mode == 'always'
+    ):
+        trigger_hook(output_data)
+
+
+asyncio.run(run(command))
\ No newline at end of file

hooks/migrations/0001_initial.py

+# Generated by Django 3.1 on 2020-12-14 15:48
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('projector', '0040_auto_20201214_1048'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Hook',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('description', models.TextField(blank=True)),
+                ('endpoint', models.SlugField(help_text='This will be the endpoint to target', max_length=256)),
+                ('hook_type', models.CharField(choices=[('test', 'Test (only logs)'), ('email', 'Send Email')], default='test', max_length=64)),
+                ('configuration', models.JSONField()),
+                ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='projector.project')),
+            ],
+        ),
+    ]

hooks/migrations/0002_hookaccesslog.py

+# Generated by Django 3.1 on 2020-12-14 19:14
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('hooks', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='HookAccessLog',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('timestamp', models.DateTimeField(auto_now_add=True)),
+                ('request_ip', models.GenericIPAddressField(blank=True, null=True)),
+                ('payload', models.TextField(blank=True)),
+                ('status', models.CharField(choices=[('OK', 'Ran Successfully'), ('ERR', 'Encountered Error')], default='OK', max_length=16)),
+                ('log', models.TextField(blank=True)),
+                ('hook', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='hooks.hook')),
+            ],
+        ),
+    ]

hooks/migrations/0003_auto_20201216_1435.py

+# Generated by Django 3.1 on 2020-12-16 19:35
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('hooks', '0002_hookaccesslog'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='hookaccesslog',
+            name='status',
+            field=models.CharField(choices=[('OK', 'Success'), ('ERR', 'Error')], default='OK', max_length=16),
+        ),
+    ]

hooks/migrations/0004_auto_20201216_1604.py

+# Generated by Django 3.1 on 2020-12-16 21:04
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('projector', '0042_auto_20201216_1604'),
+        ('hooks', '0003_auto_20201216_1435'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='hook',
+            name='project',
+            field=models.ForeignKey(help_text='Note: limited to projects for which you are a contact', on_delete=django.db.models.deletion.CASCADE, to='projector.project'),
+        ),
+    ]

hooks/models.py

+from django.db import models
+HOOK_TYPE_OPTIONS = (
+    ('test', 'Test (only logs)'),
+    ('email', 'Send Email'),
+)
+
+
+LOG_STATUS_CHOICES = (
+    ('OK', 'Success'),
+    ('ERR', 'Error'),
+)
+class Hook(models.Model):
+    project = models.ForeignKey(
+        'projector.Project', on_delete=models.CASCADE,
+        help_text="Note: limited to projects for which you are a contact"
+    )
+    description = models.TextField(blank=True)
+    endpoint = models.SlugField(
+        max_length=256,
+        help_text="This will be the endpoint to target"
+    )
+    hook_type = models.CharField(
+        choices=HOOK_TYPE_OPTIONS,
+         default='test',
+         max_length=64
+    )
+    configuration = models.JSONField()
+
+    def __str__(self):
+        return "{}/{}".format(self.project.slug, self.endpoint)
+
+class HookAccessLog(models.Model):
+    hook = models.ForeignKey('hooks.Hook', on_delete=models.CASCADE)
+    timestamp = models.DateTimeField(auto_now_add=True)
+    request_ip = models.GenericIPAddressField(blank=True, null=True)
+    payload = models.TextField(blank=True)
+    status = models.CharField(choices=LOG_STATUS_CHOICES, default='OK', max_length=16)
+    log = models.TextField(blank=True)

hooks/tests.py

+from django.test import TestCase
+
+# Create your tests here.

hooks/urls.py

+from django.urls import path
+from .views import postHook
+
+urlpatterns = [
+    path('<slug:project_slug>/<slug:endpoint>', postHook)
+]
\ No newline at end of file

hooks/views.py

+from django.shortcuts import render
+from django.http import JsonResponse, HttpResponse
+from django.views.decorators.csrf import csrf_exempt
+from django.views.decorators.http import require_http_methods
+from django.core.mail import EmailMessage
+from django.template import Template, Context
+from .models import Hook, HookAccessLog
+
+import json
+# Create your views here.
+
+def render_default(var, context_dict, *, default=''):
+    if var is None:
+        to_render = default
+    else:
+        to_render = var
+    return Template(to_render).render(Context(context_dict))
+
+
+def process_hook_email(data, hook):
+    config = hook.configuration
+    context_data = {'post_data': data, 'hook': hook}
+    subject = render_default(
+        config.get('subject', None),
+        context_data,
+        default="Hook {{hook.project.slug}}/{{hook.endpoint}} Triggered!"
+    )
+    body = render_default(
+        config.get('body', None),
+        context_data,
+        default="{{post_data|safe}}"
+    )
+    to = config['to']
+    if isinstance(to, str):
+        to = [to]
+    msg = EmailMessage(
+        subject=subject,
+        to=to,
+        body=body
+    )
+    msg.send()
+
+def process_hook_test(data, hook):
+    pass
+    
+HOOK_FN_LOOKUP = {
+    'email': process_hook_email,
+    'test': process_hook_test
+}
+
+def get_client_ip(request):
+    x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
+    if x_forwarded_for:
+        ip = x_forwarded_for.split(',')[-1]
+    else:
+        ip = request.META.get('REMOTE_ADDR')
+    return ip
+
+
+@csrf_exempt
+@require_http_methods(['POST'])
+def postHook(request, project_slug, endpoint):
+    # We'll verify there's an API key in the header before we do anything else
+    key = request.META.get('HTTP_KEY', None)
+    if key is None:
+        return JsonResponse({'error': 'No KEY header found'}, status=403)
+
+    hook = Hook.objects\
+        .filter(project__slug=project_slug, endpoint=endpoint)\
+        .select_related('project')\
+        .first()
+    
+    if hook is None:
+        return HttpResponse(status=404)
+    print(key, hook.project.slug)
+    if hook.project.slug != key:
+        return JsonResponse(
+            {'error': 'Invalid Key'},
+            status=403
+        )
+    
+    # Process...
+    log = ''
+    status = 'OK'
+    data = None
+    try:
+        data = request.body.decode('utf-8')
+        if request.META.get('CONTENT_TYPE', '') == 'application/json':
+            data = json.loads(data)
+        HOOK_FN_LOOKUP[hook.hook_type](data, hook)
+    except Exception as e:
+        log = str(e)
+        status = 'ERR'
+    
+    HookAccessLog.objects.create(
+        log=log,
+        status=status,
+        payload=request.body,
+        request_ip=get_client_ip(request),
+        hook=hook
+    )
+    return HttpResponse(status=200) if status == 'OK' else JsonResponse({'error': log},status=500)
+        
+
+
+    
+
+
+

projector/admin.py

@@ -61,11 +61,17 @@ class OwnProjectFilter(admin.SimpleListFilter):
         if self.value():
             return queryset.filter(project=self.value())
 
+class LimitProjectDropdownMixin:
+    def formfield_for_foreignkey(self, db_field, request, **kwargs):
+        if db_field.name == "project":
+            kwargs["queryset"] = Project.objects.filter(contact__user=request.user)
+        return super().formfield_for_foreignkey(db_field, request, **kwargs)
 
-class TrackedDocumentAdmin(MarkdownxModelAdmin):
+class TrackedDocumentAdmin(LimitProjectDropdownMixin, MarkdownxModelAdmin):
     list_display = ['title', 'project']
     list_filter = [OwnProjectFilter]
     fields = [
+        'project',
         'title',
         'slug',
         'body',
@@ -78,6 +84,7 @@ class TrackedDocumentAdmin(MarkdownxModelAdmin):
         qs = super().get_queryset(request)
         return qs.filter(project__contact__user=request.user)
 
+
     def save_model(self, request, obj, form, change):
         # Create revisions...
         if change:
@@ -101,7 +108,7 @@ class ProjectAdmin(MarkdownxModelAdmin):
     list_display = ['title', 'url', 'get_groups']
     fieldsets = [
         ('Project Information',     {
-            'fields': ['title', 'slug', 'icon', 'tagline', 'description', 'url', 'project_created']
+            'fields': ['title', 'slug', 'icon', 'tagline', 'description', 'url', 'project_created', 'visible', 'api_key']
         }),
         ('Project Details',    {'fields': ['scope', 'technologies', 'groups']}),
         ('Gitlab Integration',     {

projector/migrations/0040_auto_20201214_1048.py

+# Generated by Django 3.1 on 2020-12-14 15:48
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('projector', '0039_auto_20201013_1632'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='user',
+            name='first_name',
+            field=models.CharField(blank=True, max_length=150, verbose_name='first name'),
+        ),
+    ]

projector/migrations/0041_auto_20201216_1437.py

+# Generated by Django 3.1 on 2020-12-16 19:37
+
+from django.db import migrations, models
+import projector.models.projects
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('projector', '0040_auto_20201214_1048'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='project',
+            name='api_key',
+            field=models.CharField(default=projector.models.projects.make_api_key, help_text='Used for posting w/ hooks', max_length=64),
+        ),
+        migrations.AddField(
+            model_name='project',
+            name='visible',
+            field=models.BooleanField(default=True, help_text='Set this to false to not show up on the homepage.', verbose_name='Show on FAST website'),
+        ),
+        migrations.AlterField(
+            model_name='project',
+            name='slug',
+            field=models.SlugField(help_text='Must be unique. Used for URLs', max_length=250, unique=True, validators=[projector.models.projects.validate_project_slug]),
+        ),
+    ]

projector/migrations/0042_auto_20201216_1604.py

+# Generated by Django 3.1 on 2020-12-16 21:04
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('projector', '0041_auto_20201216_1437'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='trackeddocument',