Skip to content
Snippets Groups Projects
Commit 2da71998 authored by Nick Lee's avatar Nick Lee
Browse files

made script to create repos automatically

parent 3ce46562
No related branches found
No related tags found
No related merge requests found
...@@ -186,3 +186,48 @@ Runs a command or program in every folder in a given folder. ...@@ -186,3 +186,48 @@ Runs a command or program in every folder in a given folder.
>>> Running command in /u5/yc2lee/gitlab-assignments/cs349-test1/random-project/random-project >>> Running command in /u5/yc2lee/gitlab-assignments/cs349-test1/random-project/random-project
3fd99a6 another test 3fd99a6 another test
$ $
### `create-repos.py`
The `create-repos.py` script sets up repositories for a set of students. It should be
run once at the start of term. This script will, for each student:
1. Create a project/repository if one doesn't exist yet.
1. Commit a `.gitignore` to the master branch. The `.gitignore` file has one line, which is just a `#`.
1. Unprotect the master branch. Gitlab makes master branches protected by default.
1. If the `--add-students` option is given, add student as developers. This action will send an invitation
email to the student's @uwaterloo.ca email address.
#### Arguments:
* `group_name`: The only mandatory argument is the group name. The students' repositories will be
added to this group, which should be created manually from the Gitlab [Groups page](https://git.uwaterloo.ca/dashboard/groups)
before running this script.
* `--token-file TOKEN_FILE`: Same usage as in `clone.py`.
* `--add-students`: Pass this option if you want to add students. By default, students will not be added. If you set this option,
the script will ask you for your `_gitlab_session` cookie which you can get by logging in to
https://git.uwaterloo.ca and searching for `_gitlab_session` cookie. It should be on the site git.uwaterloo.ca.
The `_gitlab_session` cookie is a hash (ex. `5bcgf155e6add457d75c20db6045r9e9`).
* `--classlist CLASSLIST`: Path to your course's `.classlist` file on the linux.student.cs.uwaterloo.ca servers. The full
path is `/u/csXXX/.classlist` where XXX is your course number. The `.classlist` file is updated
automatically each midnight using enrollment data from the registrar.
* `--students STUDENTS`: You can set up repositories of a specific list of students instead of the whole class.
`STUDENTS` should be a comma separated list of student Quest IDs.
#### Examples:
1. `./create-repos.py cs123-spring2015 --token-file ~/.gitlab_token --add-students --students j4ansmith,yralimonl,t2yang`
Reads your private Gitlab token from the file `~/.gitlab_token`. Then sets up projects for j4ansmith, yralimonl, and t2yang
in the group `cs123-spring2015`. Also adds the three students to their project. Gitlab will send an invitation email
to their @uwaterloo.ca email address.
1. `./create-repos.py cs123-spring2015 --add-students --classlist /u/cs123/.classlist`
Sets up projects for all students in `/u/cs123/.classlist` at the time the script is run. Also adds students to their
project.
### `gitlab.py`
The file `gitlab.py` has some helper functions that are used by
other scripts. It doesn't do anything when run by itself.
#!/usr/bin/ssh-agent /usr/bin/python3 #!/usr/bin/ssh-agent /usr/bin/python3
import gitlab
import pprint # useful for debugging import pprint # useful for debugging
import argparse,getpass,re,time import argparse,getpass,re,time
from datetime import datetime from datetime import datetime
...@@ -10,29 +11,6 @@ import json,urllib.request ...@@ -10,29 +11,6 @@ import json,urllib.request
# Helper functions # Helper functions
# #
# gitlab_query makes a request to https://git.uwaterloo.ca/api/v3/
# and returns the JSON data as a Python object
# Input: query: Part of URL after the URL above
# Returns: A python object
def gitlab_query(query):
try:
req = urllib.request.Request(url="https://git.uwaterloo.ca/api/v3/" + query,\
headers={'PRIVATE-TOKEN': private_token})
with urllib.request.urlopen(req) as f:
json_string = f.read().decode('utf-8')
try:
python_object = json.loads(json_string)
except Exception as e:
print(json_string)
print("Error occurred trying to interpret above data as JSON.")
print("Error message: %s" % str(e))
sys.exit(1)
return python_object
except Exception as e:
print("Error occurred trying to access https://git.uwaterloo.ca/api/v3/" + query)
print("Error message: %s" % str(e))
sys.exit(1)
# Checks if s contains a valid date and time format. If it is # Checks if s contains a valid date and time format. If it is
# valid, return it as a datetime object. Otherwise, raises an # valid, return it as a datetime object. Otherwise, raises an
# error. # error.
...@@ -91,18 +69,7 @@ else: ...@@ -91,18 +69,7 @@ else:
students = None students = None
# Read private token from keyboard or from file # Read private token from keyboard or from file
if token_file == "/dev/stdin": gitlab.set_private_token(token_file)
print("You can get your Gitlab private token from https://git.uwaterloo.ca/profile/account")
private_token = getpass.getpass("Please enter your Gitlab private token:")
else:
try:
token_file_handle = open(token_file, 'r')
private_token = token_file_handle.readline().strip()
token_file_handle.close
except Exception as e:
print("Error occurred trying to read private token from file %s" % token_file)
print("Error message: %s" % str(e))
sys.exit(1)
# for debugging # for debugging
# print("group_to_clone=%s" % group_to_clone) # print("group_to_clone=%s" % group_to_clone)
...@@ -119,25 +86,7 @@ else: ...@@ -119,25 +86,7 @@ else:
# #
print("Getting ID of group %s." % group_to_clone) print("Getting ID of group %s." % group_to_clone)
group_id = gitlab.get_group_id(group_to_clone)
group_id = None
groups_data = gitlab_query("groups")
for group in groups_data:
if group['name'] == group_to_clone:
group_id = group['id']
break
if group_id == None:
print("Could not find group %s." % group_to_clone)
print("The groups that are available are:")
name_width = 20
print(os.linesep)
print("\t%s Description" % ("Name".ljust(name_width)))
print("\t%s ---------------" % ("-" * name_width))
for group in groups_data:
print("\t%s %s" % (group['name'].ljust(name_width), group['description']))
print(os.linesep)
sys.exit(1)
print("Found group %s which has ID %d" % (group_to_clone, group_id)) print("Found group %s which has ID %d" % (group_to_clone, group_id))
...@@ -152,7 +101,7 @@ print("Found group %s which has ID %d" % (group_to_clone, group_id)) ...@@ -152,7 +101,7 @@ print("Found group %s which has ID %d" % (group_to_clone, group_id))
print("Getting git repo URLs in group %s (id %d)." % (group_to_clone, group_id)) print("Getting git repo URLs in group %s (id %d)." % (group_to_clone, group_id))
group_to_clone_data = gitlab_query("groups/%d" % group_id) group_to_clone_data = gitlab.request("groups/%d" % group_id)
projects_data = group_to_clone_data['projects'] projects_data = group_to_clone_data['projects']
all_usernames = [] all_usernames = []
urls = [] urls = []
...@@ -230,7 +179,7 @@ for url_info in urls: ...@@ -230,7 +179,7 @@ for url_info in urls:
# Find the latest push that's on or before revert_date # Find the latest push that's on or before revert_date
ontime_push_time = None ontime_push_time = None
ontime_commit = None ontime_commit = None
project_events = gitlab_query('projects/%s/events' % url_info['project_id']) project_events = gitlab.request('projects/%s/events' % url_info['project_id'])
for event in project_events: for event in project_events:
# Only care about project events that are pushes to master branch # Only care about project events that are pushes to master branch
if event['action_name'] in ['pushed to', 'pushed new'] and event['data']['ref'] == 'refs/heads/master': if event['action_name'] in ['pushed to', 'pushed new'] and event['data']['ref'] == 'refs/heads/master':
......
#!/usr/bin/python3
import time
import argparse,getpass,re
import sys,subprocess,os
import json,urllib.request
import gitlab
# Parse command-line arguments.
parser = argparse.ArgumentParser(description="This script is used to create student repositories.")
parser.add_argument('group_name', help="The name of the Gitlab group to create projects in.")
parser.add_argument('--token-file', default="/dev/stdin",
help="Path to file containing your Gitlab private token. Default is to read from standard input.")
parser.add_argument('--add-students', action='store_true',
help="By default, students will not be added to their repos. Set this option to add them, which will email them too.")
students_arg_group = parser.add_mutually_exclusive_group()
students_arg_group.add_argument('--classlist', nargs=1, help="Path to your course's .classlist file on the student.cs Linux servers.")
students_arg_group.add_argument('--students', help="A comma separated list of student Quest IDs. Create repositories for these students only.")
args = parser.parse_args()
# save command line argument inputs in variables
group_name = args.group_name
token_file = args.token_file
add_students = args.add_students
# Read private token from keyboard or from file
gitlab.set_private_token(token_file)
# If adding students, read gitlab session cookie
if add_students:
print("You want students to be added to their project/repository.")
print("This script adds students by interfacing with the git.uwaterloo.ca website directly.")
print("Please login to https://git.uwaterloo.ca and enter your _gitlab_session cookie from git.uwaterloo.ca below.")
gitlab_session_cookie = getpass.getpass("git.uwaterloo.ca _gitlab_session cookie value:")
print(gitlab_session_cookie)
# Students we will create repositories for
students = []
if args.students:
students = list(map(lambda s:s.strip(), args.students.split(',')))
students = list(filter(lambda s: s and not s.isspace(), students))
elif args.classlist:
classlist_regex = re.compile('^[0-9]{8}:([a-z0-9]+):')
classlist_file = args.classlist[0]
for line in open(classlist_file, 'r'):
match = classlist_regex.match(line)
if match != None:
userid = match.group(1)
userid = userid[0:8]
students.append(userid)
# Create a hash mapping student usernames to the id of their project/repo
# This should be empty. If not, it means some projects have already
# been created.
group_id = gitlab.get_group_id(group_name)
group_data = gitlab.request("groups/%d" % group_id)
projects_data = group_data['projects']
project_ids = {}
for project in projects_data:
username = project['ssh_url_to_repo'].rsplit('/',1)[-1][:-4]
project_ids[username] = project['id']
# Begin processing students
print("Processing %d total students." % len(students))
for student in students:
print(os.linesep)
print('-' * 60)
print("> Processing %s" % student)
# Create project/repo for students who do not have one yet.
if student not in project_ids:
# Student doesn't have a project/repo yet. Create it
print("> %s doesn't have a project/repo yet. Creating it now." % student)
new_project = gitlab.request('projects', post_hash={'name':student, 'namespace_id':group_id, 'visibility_level':0})
project_ids[student] = new_project['id']
print("> Created new project with id %d" % new_project['id'])
else:
print("> %s already has a project (id %d). Not creating it again." % (student, project_ids[student]))
# Create master branch if it doesn't exist yet
existing_branches = gitlab.request('projects/%d/repository/branches' % project_ids[student])
master_branch_exists = False
for branch in existing_branches:
if branch['name'] == 'master':
master_branch_exists = True
if not master_branch_exists:
print("> master branch doesn't exist for %s. Creating it." % student)
gitlab.request('projects/%d/repository/files' % project_ids[student],
post_hash={'file_path':".gitignore", 'branch_name':"master", 'content':"#\n", 'commit_message':"Initial commit to create master branch"})
# Wait for master branch to become protected. Gitlab seems to have a delay on protecting the
# master branch when it's created.
while True:
master_branch_info = gitlab.request('/projects/%d/repository/branches/master' % project_ids[student], quit_on_error=False)
if master_branch_info and master_branch_info['protected']:
print("> Newly created master branch has become protected.")
break
print("> Waiting for Gitlab to make newly created master branch protected.")
time.sleep(1) # Don't spam Gitlab website
else:
print("> master branch already exists for %s. Not creating it." % student)
# Turn off master branch protection (on by default). At this point
# in the code, we have created master branch if it doesn't exist.
# So master branch should exist. Also, if master is already unprotected,
# then this operation does nothing (it's idempotent).
print("> Unprotecting master branch.")
gitlab.request('/projects/%d/repository/branches/master/unprotect' % project_ids[student], http_method='PUT')
# The repo is now set up with an unprotected master branch.
# Do email invitation if user wants to do that.
if add_students:
print("> Adding student to project/repository.")
# Step 1: Go to project_members web page and get authenticity token.
print("> Getting authenticity token from project_members page.")
authenticity_token = None
req = urllib.request.Request("https://git.uwaterloo.ca/%s/%s/project_members" % (group_name, student),
headers={'Cookie': "_gitlab_session=%s"%gitlab_session_cookie})
with urllib.request.urlopen(req) as f:
project_members_html = f.read().decode('utf-8')
for line in project_members_html.splitlines():
match = re.search(r'<input name="authenticity_token" type="hidden" value="([^"]+)" />', line)
if match:
authenticity_token = match.group(1)
break
# Step 2: Make the post request to invite by email
if authenticity_token:
student_email = "%s@uwaterloo.ca" % student
print("> Got authenticity token.")
print("> Sending invitation email to %s" % student_email)
post_data = urllib.parse.urlencode({'authenticity_token':authenticity_token,'user_ids':student_email,'access_level':30}).encode('ascii')
add_student_post = urllib.request.Request("https://git.uwaterloo.ca/%s/%s/project_members" % (group_name, student),
headers={'Cookie': "_gitlab_session=%s"%gitlab_session_cookie},
data=post_data, method='POST')
urllib.request.urlopen(add_student_post)
else:
print("> Could not get add student %s to repo!" % student)
print("> Done processing %s." % student)
time.sleep(3) # Put in a bit of a delay so that git.uwaterloo.ca isn't hammered
#!/usr/bin/python3
import sys,getpass
import json,urllib.request
private_token = ''
# request makes a request to https://git.uwaterloo.ca/api/v3/
# and returns the JSON data as a Python object
# Input: query: Part of URL after the URL above
# post_hash: A dictionary of data to send in a POST request
# query_headers: Any headers you want to send as part of the request
# quit_on_error: If True, will quit program on error. If False, will
# return False on error instead
# Returns: A python object
def request(query, post_hash={}, query_headers={}, http_method=None, quit_on_error=True):
try:
if 'PRIVATE-TOKEN' not in query_headers:
query_headers['PRIVATE-TOKEN'] = private_token
post_data = urllib.parse.urlencode(post_hash).encode('ascii') if post_hash else None
req = urllib.request.Request(url="https://git.uwaterloo.ca/api/v3/" + query,
data=post_data,
headers=query_headers,
method=http_method)
with urllib.request.urlopen(req) as f:
json_string = f.read().decode('utf-8')
try:
python_object = json.loads(json_string)
except Exception as e:
print(json_string)
print("Error occurred trying to interpret above data as JSON.")
print("Error message: %s" % str(e))
if quit_on_error:
sys.exit(1)
else:
return False
return python_object
except Exception as e:
print("Error occurred trying to access https://git.uwaterloo.ca/api/v3/" + query)
print("Error %s message: %s" % (type(e).__name__, str(e)))
if quit_on_error:
sys.exit(1)
else:
return False
# Read private token from token_file. Mutates the global private_token
# above and returns it too.
def set_private_token(token_file):
global private_token
if token_file == "/dev/stdin":
print("You can get your Gitlab private token from https://git.uwaterloo.ca/profile/account")
private_token = getpass.getpass("Please enter your Gitlab private token:")
return private_token
else:
try:
token_file_handle = open(token_file, 'r')
private_token = token_file_handle.readline().strip()
token_file_handle.close()
return private_token
except Exception as e:
print("Error occurred trying to read private token from file %s" % token_file)
print("Error message: %s" % str(e))
sys.exit(1)
# Returns the group id (an integer) of group_name. If group_name could
# not be found, prints the groups available and exit.
def get_group_id(group_name):
groups_data = request('groups')
for group in groups_data:
if group['name'] == group_name:
return group['id']
# could not find a group with the given name
print("Could not find group %s." % group_to_clone)
print("The groups that are available are:")
name_width = 20
print(os.linesep)
print("\t%s Description" % ("Name".ljust(name_width)))
print("\t%s ---------------" % ("-" * name_width))
for group in groups_data:
print("\t%s %s" % (group['name'].ljust(name_width), group['description']))
print(os.linesep)
sys.exit(1)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment