made script to create repos automatically

README.md

@@ -186,3 +186,48 @@ Runs a command or program in every folder in a given folder.
         >>> Running command in /u5/yc2lee/gitlab-assignments/cs349-test1/random-project/random-project
         3fd99a6 another test
         $
+
+### `create-repos.py`
+
+The `create-repos.py` script sets up repositories for a set of students. It should be
+run once at the start of term. This script will, for each student:
+
+1. Create a project/repository if one doesn't exist yet.
+1. Commit a `.gitignore` to the master branch. The `.gitignore` file has one line, which is just a `#`.
+1. Unprotect the master branch. Gitlab makes master branches protected by default.
+1. If the `--add-students` option is given, add student as developers. This action will send an invitation
+   email to the student's @uwaterloo.ca email address.
+
+#### Arguments:
+
+* `group_name`: The only mandatory argument is the group name. The students' repositories will be
+  added to this group, which should be created manually from the Gitlab [Groups page](https://git.uwaterloo.ca/dashboard/groups)
+  before running this script.
+* `--token-file TOKEN_FILE`: Same usage as in `clone.py`.
+* `--add-students`: Pass this option if you want to add students. By default, students will not be added. If you set this option,
+                    the script will ask you for your `_gitlab_session` cookie which you can get by logging in to
+                    https://git.uwaterloo.ca and searching for `_gitlab_session` cookie. It should be on the site git.uwaterloo.ca.
+                    The `_gitlab_session` cookie is a hash (ex. `5bcgf155e6add457d75c20db6045r9e9`).
+* `--classlist CLASSLIST`: Path to your course's `.classlist` file on the linux.student.cs.uwaterloo.ca servers. The full
+                           path is `/u/csXXX/.classlist` where XXX is your course number. The `.classlist` file is updated
+                           automatically each midnight using enrollment data from the registrar.
+* `--students STUDENTS`: You can set up repositories of a specific list of students instead of the whole class.
+                         `STUDENTS` should be a comma separated list of student Quest IDs.
+
+#### Examples:
+
+1. `./create-repos.py cs123-spring2015 --token-file ~/.gitlab_token --add-students --students j4ansmith,yralimonl,t2yang`
+
+   Reads your private Gitlab token from the file `~/.gitlab_token`. Then sets up projects for j4ansmith, yralimonl, and t2yang
+   in the group `cs123-spring2015`. Also adds the three students to their project. Gitlab will send an invitation email
+   to their @uwaterloo.ca email address.
+
+1. `./create-repos.py cs123-spring2015 --add-students --classlist /u/cs123/.classlist`
+
+   Sets up projects for all students in `/u/cs123/.classlist` at the time the script is run. Also adds students to their
+   project.
+
+### `gitlab.py`
+
+The file `gitlab.py` has some helper functions that are used by
+other scripts. It doesn't do anything when run by itself.

clone.py

 #!/usr/bin/ssh-agent /usr/bin/python3
 
+import gitlab
 import pprint # useful for debugging
 import argparse,getpass,re,time
 from datetime import datetime
@@ -10,29 +11,6 @@ import json,urllib.request
 # Helper functions
 #
 
-# gitlab_query makes a request to https://git.uwaterloo.ca/api/v3/
-# and returns the JSON data as a Python object
-# Input: query: Part of URL after the URL above
-# Returns: A python object
-def gitlab_query(query):
-    try:
-        req = urllib.request.Request(url="https://git.uwaterloo.ca/api/v3/" + query,\
-                                     headers={'PRIVATE-TOKEN': private_token})
-        with urllib.request.urlopen(req) as f:
-            json_string = f.read().decode('utf-8')
-            try:
-                python_object = json.loads(json_string)
-            except Exception as e:
-                print(json_string)
-                print("Error occurred trying to interpret above data as JSON.")
-                print("Error message: %s" % str(e))
-                sys.exit(1)
-            return python_object
-    except Exception as e:
-        print("Error occurred trying to access https://git.uwaterloo.ca/api/v3/" + query)
-        print("Error message: %s" % str(e))
-        sys.exit(1)
-
 # Checks if s contains a valid date and time format. If it is
 # valid, return it as a datetime object. Otherwise, raises an
 # error.
@@ -91,18 +69,7 @@ else:
     students = None
 
 # Read private token from keyboard or from file
-if token_file == "/dev/stdin":
-    print("You can get your Gitlab private token from https://git.uwaterloo.ca/profile/account")
-    private_token = getpass.getpass("Please enter your Gitlab private token:")
-else:
-    try:
-        token_file_handle = open(token_file, 'r')
-        private_token = token_file_handle.readline().strip()
-        token_file_handle.close
-    except Exception as e:
-        print("Error occurred trying to read private token from file %s" % token_file)
-        print("Error message: %s" % str(e))
-        sys.exit(1)
+gitlab.set_private_token(token_file)
 
 # for debugging
 # print("group_to_clone=%s" % group_to_clone)
@@ -119,25 +86,7 @@ else:
 #
 
 print("Getting ID of group %s." % group_to_clone)
-
-group_id = None
-groups_data = gitlab_query("groups")
-for group in groups_data:
-    if group['name'] == group_to_clone:
-        group_id = group['id']
-        break
-if group_id == None:
-    print("Could not find group %s." % group_to_clone)
-    print("The groups that are available are:")
-    name_width = 20
-    print(os.linesep)
-    print("\t%s   Description" % ("Name".ljust(name_width)))
-    print("\t%s   ---------------" % ("-" * name_width))
-    for group in groups_data:
-        print("\t%s   %s" % (group['name'].ljust(name_width), group['description']))
-    print(os.linesep)
-    sys.exit(1)
-
+group_id = gitlab.get_group_id(group_to_clone)
 print("Found group %s which has ID %d" % (group_to_clone, group_id))
 
 
@@ -152,7 +101,7 @@ print("Found group %s which has ID %d" % (group_to_clone, group_id))
 
 print("Getting git repo URLs in group %s (id %d)." % (group_to_clone, group_id))
 
-group_to_clone_data = gitlab_query("groups/%d" % group_id)
+group_to_clone_data = gitlab.request("groups/%d" % group_id)
 projects_data = group_to_clone_data['projects']
 all_usernames = []
 urls = []
@@ -230,7 +179,7 @@ for url_info in urls:
         # Find the latest push that's on or before revert_date
         ontime_push_time = None
         ontime_commit    = None
-        project_events = gitlab_query('projects/%s/events' % url_info['project_id'])
+        project_events = gitlab.request('projects/%s/events' % url_info['project_id'])
         for event in project_events:
             # Only care about project events that are pushes to master branch
             if event['action_name'] in ['pushed to', 'pushed new'] and event['data']['ref'] == 'refs/heads/master':

create-repos.py

+#!/usr/bin/python3
+
+import time
+import argparse,getpass,re
+import sys,subprocess,os
+import json,urllib.request
+import gitlab
+
+# Parse command-line arguments.
+parser = argparse.ArgumentParser(description="This script is used to create student repositories.")
+parser.add_argument('group_name', help="The name of the Gitlab group to create projects in.")
+parser.add_argument('--token-file', default="/dev/stdin",
+                    help="Path to file containing your Gitlab private token. Default is to read from standard input.")
+parser.add_argument('--add-students', action='store_true',
+                    help="By default, students will not be added to their repos. Set this option to add them, which will email them too.")
+students_arg_group = parser.add_mutually_exclusive_group()
+students_arg_group.add_argument('--classlist', nargs=1, help="Path to your course's .classlist file on the student.cs Linux servers.")
+students_arg_group.add_argument('--students', help="A comma separated list of student Quest IDs. Create repositories for these students only.")
+args = parser.parse_args()
+
+# save command line argument inputs in variables
+group_name = args.group_name
+token_file = args.token_file
+add_students = args.add_students
+
+# Read private token from keyboard or from file
+gitlab.set_private_token(token_file)
+
+# If adding students, read gitlab session cookie
+if add_students:
+    print("You want students to be added to their project/repository.")
+    print("This script adds students by interfacing with the git.uwaterloo.ca website directly.")
+    print("Please login to https://git.uwaterloo.ca and enter your _gitlab_session cookie from git.uwaterloo.ca below.")
+    gitlab_session_cookie = getpass.getpass("git.uwaterloo.ca _gitlab_session cookie value:")
+    print(gitlab_session_cookie)
+
+# Students we will create repositories for
+students = []
+if args.students:
+    students = list(map(lambda s:s.strip(), args.students.split(',')))
+    students = list(filter(lambda s: s and not s.isspace(), students))
+elif args.classlist:
+    classlist_regex = re.compile('^[0-9]{8}:([a-z0-9]+):')
+    classlist_file = args.classlist[0]
+    for line in open(classlist_file, 'r'):
+        match = classlist_regex.match(line)
+        if match != None:
+            userid = match.group(1)
+            userid = userid[0:8]
+            students.append(userid)
+
+# Create a hash mapping student usernames to the id of their project/repo
+# This should be empty. If not, it means some projects have already
+# been created.
+group_id = gitlab.get_group_id(group_name)
+group_data = gitlab.request("groups/%d" % group_id)
+projects_data = group_data['projects']
+project_ids = {}
+for project in projects_data:
+    username = project['ssh_url_to_repo'].rsplit('/',1)[-1][:-4]
+    project_ids[username] = project['id']
+
+# Begin processing students
+print("Processing %d total students." % len(students))
+for student in students:
+    print(os.linesep)
+    print('-' * 60)
+    print("> Processing %s" % student)
+    
+    # Create project/repo for students who do not have one yet.
+    if student not in project_ids:
+        # Student doesn't have a project/repo yet. Create it
+        print("> %s doesn't have a project/repo yet. Creating it now." % student)
+        new_project = gitlab.request('projects', post_hash={'name':student, 'namespace_id':group_id, 'visibility_level':0})
+        project_ids[student] = new_project['id']
+        print("> Created new project with id %d" % new_project['id'])
+    else:
+        print("> %s already has a project (id %d). Not creating it again." % (student, project_ids[student]))
+
+    # Create master branch if it doesn't exist yet
+    existing_branches = gitlab.request('projects/%d/repository/branches' % project_ids[student])
+    master_branch_exists = False
+    for branch in existing_branches:
+        if branch['name'] == 'master':
+            master_branch_exists = True
+    if not master_branch_exists:
+        print("> master branch doesn't exist for %s. Creating it." % student)
+        gitlab.request('projects/%d/repository/files' % project_ids[student],
+                       post_hash={'file_path':".gitignore", 'branch_name':"master", 'content':"#\n", 'commit_message':"Initial commit to create master branch"})
+        # Wait for master branch to become protected. Gitlab seems to have a delay on protecting the
+        # master branch when it's created.
+        while True:
+            master_branch_info = gitlab.request('/projects/%d/repository/branches/master' % project_ids[student], quit_on_error=False)
+            if master_branch_info and master_branch_info['protected']:
+                print("> Newly created master branch has become protected.")
+                break
+            print("> Waiting for Gitlab to make newly created master branch protected.")
+            time.sleep(1) # Don't spam Gitlab website
+    else:
+        print("> master branch already exists for %s. Not creating it." % student)
+
+    # Turn off master branch protection (on by default). At this point
+    # in the code, we have created master branch if it doesn't exist.
+    # So master branch should exist. Also, if master is already unprotected,
+    # then this operation does nothing (it's idempotent).
+    print("> Unprotecting master branch.")
+    gitlab.request('/projects/%d/repository/branches/master/unprotect' % project_ids[student], http_method='PUT')
+        
+    # The repo is now set up with an unprotected master branch.
+    # Do email invitation if user wants to do that.
+    if add_students:
+        print("> Adding student to project/repository.")
+
+        # Step 1: Go to project_members web page and get authenticity token.
+        print("> Getting authenticity token from project_members page.")
+        authenticity_token = None
+        req = urllib.request.Request("https://git.uwaterloo.ca/%s/%s/project_members" % (group_name, student),
+                                     headers={'Cookie': "_gitlab_session=%s"%gitlab_session_cookie})
+        with urllib.request.urlopen(req) as f:
+            project_members_html = f.read().decode('utf-8')
+            for line in project_members_html.splitlines():
+                match = re.search(r'<input name="authenticity_token" type="hidden" value="([^"]+)" />', line)
+                if match:
+                    authenticity_token = match.group(1)
+                    break
+
+        # Step 2: Make the post request to invite by email
+        if authenticity_token:
+            student_email = "%s@uwaterloo.ca" % student
+            print("> Got authenticity token.")
+            print("> Sending invitation email to %s" % student_email)
+            post_data = urllib.parse.urlencode({'authenticity_token':authenticity_token,'user_ids':student_email,'access_level':30}).encode('ascii')
+            add_student_post = urllib.request.Request("https://git.uwaterloo.ca/%s/%s/project_members" % (group_name, student),
+                                                      headers={'Cookie': "_gitlab_session=%s"%gitlab_session_cookie},
+                                                      data=post_data, method='POST')
+            urllib.request.urlopen(add_student_post)
+        else:
+            print("> Could not get add student %s to repo!" % student)
+
+    print("> Done processing %s." % student)
+    time.sleep(3) # Put in a bit of a delay so that git.uwaterloo.ca isn't hammered

gitlab.py

+#!/usr/bin/python3
+
+import sys,getpass
+import json,urllib.request
+
+private_token = ''
+
+# request makes a request to https://git.uwaterloo.ca/api/v3/
+# and returns the JSON data as a Python object
+# Input: query: Part of URL after the URL above
+#        post_hash: A dictionary of data to send in a POST request
+#        query_headers: Any headers you want to send as part of the request
+#        quit_on_error: If True, will quit program on error. If False, will
+#                       return False on error instead
+# Returns: A python object
+def request(query, post_hash={}, query_headers={}, http_method=None, quit_on_error=True):
+    try:
+        if 'PRIVATE-TOKEN' not in query_headers:
+            query_headers['PRIVATE-TOKEN'] = private_token
+        post_data = urllib.parse.urlencode(post_hash).encode('ascii') if post_hash else None
+        req = urllib.request.Request(url="https://git.uwaterloo.ca/api/v3/" + query,
+                                     data=post_data,
+                                     headers=query_headers,
+                                     method=http_method)
+        with urllib.request.urlopen(req) as f:
+            json_string = f.read().decode('utf-8')
+            try:
+                python_object = json.loads(json_string)
+            except Exception as e:
+                print(json_string)
+                print("Error occurred trying to interpret above data as JSON.")
+                print("Error message: %s" % str(e))
+                if quit_on_error:
+                    sys.exit(1)
+                else:
+                    return False
+            return python_object
+    except Exception as e:
+        print("Error occurred trying to access https://git.uwaterloo.ca/api/v3/" + query)
+        print("Error %s message: %s" % (type(e).__name__, str(e)))
+        if quit_on_error:
+            sys.exit(1)
+        else:
+            return False
+
+# Read private token from token_file. Mutates the global private_token
+# above and returns it too.
+def set_private_token(token_file):
+    global private_token
+    if token_file == "/dev/stdin":
+        print("You can get your Gitlab private token from https://git.uwaterloo.ca/profile/account")
+        private_token = getpass.getpass("Please enter your Gitlab private token:")
+        return private_token
+    else:
+        try:
+            token_file_handle = open(token_file, 'r')
+            private_token = token_file_handle.readline().strip()
+            token_file_handle.close()
+            return private_token
+        except Exception as e:
+            print("Error occurred trying to read private token from file %s" % token_file)
+            print("Error message: %s" % str(e))
+            sys.exit(1)
+
+# Returns the group id (an integer) of group_name. If group_name could
+# not be found, prints the groups available and exit.
+def get_group_id(group_name):
+    groups_data = request('groups')
+    for group in groups_data:
+        if group['name'] == group_name:
+            return group['id']
+    # could not find a group with the given name
+    print("Could not find group %s." % group_to_clone)
+    print("The groups that are available are:")
+    name_width = 20
+    print(os.linesep)
+    print("\t%s   Description" % ("Name".ljust(name_width)))
+    print("\t%s   ---------------" % ("-" * name_width))
+    for group in groups_data:
+        print("\t%s   %s" % (group['name'].ljust(name_width), group['description']))
+    print(os.linesep)
+    sys.exit(1)