Commit 7a24c2b1 authored by Mirko Vucicevich's avatar Mirko Vucicevich
Browse files

Added hooks and finalized tracked document API

parent e15eb771
# FAST Hooks
Mirko got it into his head this would be a good idea, and thus it now exists. FAST now supports very basic hooks which trigger various actions. At the moment this is only being used for sending emails on failed cron jobs.
A python3 script is provided in this directory `fast_hook`, which can be run as follows:
```
fast_hook -k <project key> -e fast/endpoint -v key value -v key2 value2 '<command>'
```
The project keys are available in the admin panel for a given fast projct, and hook endpoints are `<project_slug>/<endpoint>`.
The script, by default, will send `stderr` to the endpoint. Any `-v` flags will be included in the JSON data posted to the hook. If you'd like to include `stdout` in the post add an `-o` flag to the command. See `fast_hook --help` for more info.
## Setting Up Hooks
Right now the hooks are super limited in what they can do and who can set them up. They only support email, and configuration needs to be done via manual JSON.
> This workflow may be improved in the future if the feature is used my multiple parties. Notably adding a UI for editing the stupid config.
Note: **you can only edit hooks for projects you've been set as a contact for**. This was mostly done to reduce the amount of noise in the UI.
Here's an example hook setup:
```
project: fast
endpoint: notify
type: email
configuration:
{
"to": "mvucicev@uwaterloo.ca",
"body": "The command {{post_data.command|safe}} failed with the following output: {{post_data.stderr|safe}}",
"subject": "Alert: [{{title}}] process failed."
}
```
> body and subject can be left blank and some defaults will be used.
Then hit the hook with the following:
```
fast_hook -v title 'Test Command' -k <key> -e fast/notify 'ls /zzz'
```
The command should fail, and the email
## I don't want to use your stupid wrapper script
That's fine! You can just use curl or something:
```
curl -X POST https://fast.uwaterloo.ca/hooks/<project>/<endpoint> -H 'key:<key>'
```
## Why didn't you just write a wrapper script that sent you emails or used CRON's emailing system?
Hopefully this can be extended -- it should allow for way more options
## What if I want a hook endpoint not attached to a specific project?
I recommend creating a new "invisible" project -- create a new project and simply set "Show on FAST Website" to false.
from django.contrib import admin
from .models import Hook, HookAccessLog
from projector.admin import OwnProjectFilter, LimitProjectDropdownMixin
from django.db.models import F
# Register your models here.
class HookFilter(admin.SimpleListFilter):
title = "Hook"
parameter_name = 'hook'
def lookups(self, request, model_admin):
return [(x[0], "{}/{}".format(x[1], x[2]))
for x in model_admin.get_queryset(request)
.filter(hook__project__contact__user=request.user)
.values_list('hook__pk', 'hook__project__slug', 'hook__endpoint')
]
def queryset(self, request, queryset):
if self.value():
return queryset.filter(hook=self.value())
class HookProjectFilter(admin.SimpleListFilter):
title = "Project"
parameter_name = 'project'
def lookups(self, request, model_admin):
return model_admin.get_queryset(request)\
.filter(hook__project__contact__user=request.user)\
.values_list('hook__project__pk', 'hook__project__title')
def queryset(self, request, queryset):
if self.value():
return queryset.filter(hook__project=self.value())
@admin.register(Hook)
class HookAdmin(LimitProjectDropdownMixin, admin.ModelAdmin):
list_filter = [OwnProjectFilter]
list_display = ['endpoint', 'project_title']
def project_title(self, obj):
return obj.project.title
project_title.admin_order_field = 'project__title'
def get_queryset(self, request):
qs = super().get_queryset(request)
return qs.filter(project__contact__user=request.user).select_related('project')
@admin.register(HookAccessLog)
class HookAccessLogAdmin(admin.ModelAdmin):
list_filter = [HookProjectFilter, HookFilter]
list_display = ['timestamp', 'status', 'project', 'endpoint']
def has_change_permission(*args, **kwargs):
return False
def get_queryset(self, request):
qs = super().get_queryset(request)
return qs.filter(hook__project__contact__user=request.user)\
.annotate(project=F('hook__project__title'))\
.annotate(endpoint=F('hook__endpoint'))
def project(self, obj):
return obj.project
def endpoint(self, obj):
return obj.endpoint
from django.apps import AppConfig
class HooksConfig(AppConfig):
name = 'hooks'
#!/usr/bin/env python3
import asyncio
import argparse
import json
import re
import urllib.request, urllib.error
parser = argparse.ArgumentParser(description='Wraps a command, sends output to a fast hook')
parser.add_argument(
'--key', '-k', required=True,
help='Your project key (found in fast admin panel for project)'
)
parser.add_argument(
'--add-value', '-v', nargs=2, action="append",
help="Add extra values to your output json -- eg. -v title Hello -v type string"
)
parser.add_argument(
'--endpoint', '-e', required=True,
help="The hook endpoint -- project_slug/endpoint"
)
parser.add_argument(
'--include-stdout', '-o', default=False, action='store_true',
help="The default setup won't include stdout in the POST data"
)
parser.add_argument(
'--mode', '-m', default='err',
help=(
"What mode to run in. Options are err, out, or always.\n"
"- err: Only trigger on non-null stderr\n"
"- out: Trigger on non-null stdout (includes stdout automatically)\n"
"- always: Always trigger even if no output"
)
)
parser.add_argument(
'--hooks-url', '-u', default='https://fast.uwaterloo.ca/hooks',
help="Defaults to https://fast.uwaterloo.ca/hooks"
)
parser.add_argument('command', nargs='*')
args = parser.parse_args()
# Clean up and validate args
if not re.match(r'^[a-z0-9-_]+\/[a-z0-9-_]+$', args.endpoint):
print("Invalid endpoint. Must be in the form 'project-slug/endpoint'")
exit(1)
if not len(args.command):
print("No command found.")
exit(1)
command = " ".join(args.command)
def trigger_hook(data):
url = f'{args.hooks_url}/{args.endpoint}'
print(f"Triggering hook @ {url}")
req = urllib.request.Request(
url,
data=json.dumps(data).encode('utf-8'),
headers={
'content-type': 'application/json',
'key': args.key
}
)
try:
urllib.request.urlopen(req)
except urllib.error.HTTPError as e:
print(f'Failed to trigger: {e}')
print(e.read().decode())
async def run(cmd):
output_data = {'status': 'OK'}
if args.add_value:
for r in args.add_value:
output_data[r[0]] = r[1]
proc = await asyncio.create_subprocess_shell(
cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE
)
stdout, stderr = await proc.communicate()
print(f'-{cmd!r} exited with {proc.returncode}-')
if stdout:
out = stdout.decode()
print(out)
if args.include_stdout or args.mode == 'out':
output_data['stdout'] = out
if stderr:
output_data['status'] = 'ERR'
err = stderr.decode()
print(f'[stderr]\n{err}')
output_data['stderr'] = err
# some quick checks
has_error = output_data.get('stderr', None) is not None
has_output = output_data.get('stdout', None) is not None
if (
args.mode == 'err' and has_error
or args.mode == 'out' and has_output
or args.mode == 'always'
):
trigger_hook(output_data)
asyncio.run(run(command))
\ No newline at end of file
# Generated by Django 3.1 on 2020-12-14 15:48
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
initial = True
dependencies = [
('projector', '0040_auto_20201214_1048'),
]
operations = [
migrations.CreateModel(
name='Hook',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('description', models.TextField(blank=True)),
('endpoint', models.SlugField(help_text='This will be the endpoint to target', max_length=256)),
('hook_type', models.CharField(choices=[('test', 'Test (only logs)'), ('email', 'Send Email')], default='test', max_length=64)),
('configuration', models.JSONField()),
('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='projector.project')),
],
),
]
# Generated by Django 3.1 on 2020-12-14 19:14
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('hooks', '0001_initial'),
]
operations = [
migrations.CreateModel(
name='HookAccessLog',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('timestamp', models.DateTimeField(auto_now_add=True)),
('request_ip', models.GenericIPAddressField(blank=True, null=True)),
('payload', models.TextField(blank=True)),
('status', models.CharField(choices=[('OK', 'Ran Successfully'), ('ERR', 'Encountered Error')], default='OK', max_length=16)),
('log', models.TextField(blank=True)),
('hook', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='hooks.hook')),
],
),
]
# Generated by Django 3.1 on 2020-12-16 19:35
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('hooks', '0002_hookaccesslog'),
]
operations = [
migrations.AlterField(
model_name='hookaccesslog',
name='status',
field=models.CharField(choices=[('OK', 'Success'), ('ERR', 'Error')], default='OK', max_length=16),
),
]
# Generated by Django 3.1 on 2020-12-16 21:04
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('projector', '0042_auto_20201216_1604'),
('hooks', '0003_auto_20201216_1435'),
]
operations = [
migrations.AlterField(
model_name='hook',
name='project',
field=models.ForeignKey(help_text='Note: limited to projects for which you are a contact', on_delete=django.db.models.deletion.CASCADE, to='projector.project'),
),
]
from django.db import models
HOOK_TYPE_OPTIONS = (
('test', 'Test (only logs)'),
('email', 'Send Email'),
)
LOG_STATUS_CHOICES = (
('OK', 'Success'),
('ERR', 'Error'),
)
class Hook(models.Model):
project = models.ForeignKey(
'projector.Project', on_delete=models.CASCADE,
help_text="Note: limited to projects for which you are a contact"
)
description = models.TextField(blank=True)
endpoint = models.SlugField(
max_length=256,
help_text="This will be the endpoint to target"
)
hook_type = models.CharField(
choices=HOOK_TYPE_OPTIONS,
default='test',
max_length=64
)
configuration = models.JSONField()
def __str__(self):
return "{}/{}".format(self.project.slug, self.endpoint)
class HookAccessLog(models.Model):
hook = models.ForeignKey('hooks.Hook', on_delete=models.CASCADE)
timestamp = models.DateTimeField(auto_now_add=True)
request_ip = models.GenericIPAddressField(blank=True, null=True)
payload = models.TextField(blank=True)
status = models.CharField(choices=LOG_STATUS_CHOICES, default='OK', max_length=16)
log = models.TextField(blank=True)
from django.test import TestCase
# Create your tests here.
from django.urls import path
from .views import postHook
urlpatterns = [
path('<slug:project_slug>/<slug:endpoint>', postHook)
]
\ No newline at end of file
from django.shortcuts import render
from django.http import JsonResponse, HttpResponse
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_http_methods
from django.core.mail import EmailMessage
from django.template import Template, Context
from .models import Hook, HookAccessLog
import json
# Create your views here.
def render_default(var, context_dict, *, default=''):
if var is None:
to_render = default
else:
to_render = var
return Template(to_render).render(Context(context_dict))
def process_hook_email(data, hook):
config = hook.configuration
context_data = {'post_data': data, 'hook': hook}
subject = render_default(
config.get('subject', None),
context_data,
default="Hook {{hook.project.slug}}/{{hook.endpoint}} Triggered!"
)
body = render_default(
config.get('body', None),
context_data,
default="{{post_data|safe}}"
)
to = config['to']
if isinstance(to, str):
to = [to]
msg = EmailMessage(
subject=subject,
to=to,
body=body
)
msg.send()
def process_hook_test(data, hook):
pass
HOOK_FN_LOOKUP = {
'email': process_hook_email,
'test': process_hook_test
}
def get_client_ip(request):
x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
if x_forwarded_for:
ip = x_forwarded_for.split(',')[-1]
else:
ip = request.META.get('REMOTE_ADDR')
return ip
@csrf_exempt
@require_http_methods(['POST'])
def postHook(request, project_slug, endpoint):
# We'll verify there's an API key in the header before we do anything else
key = request.META.get('HTTP_KEY', None)
if key is None:
return JsonResponse({'error': 'No KEY header found'}, status=403)
hook = Hook.objects\
.filter(project__slug=project_slug, endpoint=endpoint)\
.select_related('project')\
.first()
if hook is None:
return HttpResponse(status=404)
print(key, hook.project.slug)
if hook.project.slug != key:
return JsonResponse(
{'error': 'Invalid Key'},
status=403
)
# Process...
log = ''
status = 'OK'
data = None
try:
data = request.body.decode('utf-8')
if request.META.get('CONTENT_TYPE', '') == 'application/json':
data = json.loads(data)
HOOK_FN_LOOKUP[hook.hook_type](data, hook)
except Exception as e:
log = str(e)
status = 'ERR'
HookAccessLog.objects.create(
log=log,
status=status,
payload=request.body,
request_ip=get_client_ip(request),
hook=hook
)
return HttpResponse(status=200) if status == 'OK' else JsonResponse({'error': log},status=500)
......@@ -61,11 +61,17 @@ class OwnProjectFilter(admin.SimpleListFilter):
if self.value():
return queryset.filter(project=self.value())
class LimitProjectDropdownMixin:
def formfield_for_foreignkey(self, db_field, request, **kwargs):
if db_field.name == "project":
kwargs["queryset"] = Project.objects.filter(contact__user=request.user)
return super().formfield_for_foreignkey(db_field, request, **kwargs)
class TrackedDocumentAdmin(MarkdownxModelAdmin):
class TrackedDocumentAdmin(LimitProjectDropdownMixin, MarkdownxModelAdmin):
list_display = ['title', 'project']
list_filter = [OwnProjectFilter]
fields = [
'project',
'title',
'slug',
'body',
......@@ -78,6 +84,7 @@ class TrackedDocumentAdmin(MarkdownxModelAdmin):
qs = super().get_queryset(request)
return qs.filter(project__contact__user=request.user)
def save_model(self, request, obj, form, change):
# Create revisions...
if change:
......@@ -101,7 +108,7 @@ class ProjectAdmin(MarkdownxModelAdmin):
list_display = ['title', 'url', 'get_groups']
fieldsets = [
('Project Information', {
'fields': ['title', 'slug', 'icon', 'tagline', 'description', 'url', 'project_created']
'fields': ['title', 'slug', 'icon', 'tagline', 'description', 'url', 'project_created', 'visible', 'api_key']
}),
('Project Details', {'fields': ['scope', 'technologies', 'groups']}),
('Gitlab Integration', {
......
# Generated by Django 3.1 on 2020-12-14 15:48
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('projector', '0039_auto_20201013_1632'),
]
operations = [
migrations.AlterField(
model_name='user',
name='first_name',
field=models.CharField(blank=True, max_length=150, verbose_name='first name'),
),
]
# Generated by Django 3.1 on 2020-12-16 19:37
from django.db import migrations, models
import projector.models.projects
class Migration(migrations.Migration):
dependencies = [
('projector', '0040_auto_20201214_1048'),
]
operations = [
migrations.AddField(
model_name='project',
name='api_key',
field=models.CharField(default=projector.models.projects.make_api_key, help_text='Used for posting w/ hooks', max_length=64),
),
migrations.AddField(
model_name='project',
name='visible',
field=models.BooleanField(default=True, help_text='Set this to false to not show up on the homepage.', verbose_name='Show on FAST website'),
),
migrations.AlterField(
model_name='project',
name='slug',
field=models.SlugField(help_text='Must be unique. Used for URLs', max_length=250, unique=True, validators=[projector.models.projects.validate_project_slug]),
),
]
# Generated by Django 3.1 on 2020-12-16 21:04
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('projector', '0041_auto_20201216_1437'),
]
operations = [
migrations.AlterField(
model_name='trackeddocument',