add exams commands

tools/exams.py

+#!/usr/bin/env python3
+
+import click
+import csv
+import os
+import tempfile
+import shutil
+import subprocess
+
+from tqdm import tqdm
+
+
+@click.group()
+def exams():
+    """Commands for generating exam variants and importing into Moodle."""
+
+def compile(frame_file, output_dir, output_name, env, verbose):
+    tmp = tempfile.mkdtemp()
+    cmd = ['pdflatex', f'-output-directory={tmp}', f'{frame_file}']
+    subenv = os.environ.copy()
+    env = {k:v.encode('latin1') for k, v in env.items()}
+    subenv.update(env)
+    if verbose:
+        cmd.append('-interaction=nonstopmode')
+        subprocess.run(cmd, env=subenv)
+    else:
+        cmd.append('-interaction=batchmode')
+        subprocess.run(cmd, env=subenv,
+                       stdout=subprocess.DEVNULL,
+                       stderr=subprocess.DEVNULL)
+    frm = os.path.join(tmp, frame_file.replace('.tex', '.pdf'))
+    to = os.path.join(output_dir, f'{output_name}.pdf')
+    shutil.copy(frm, to)
+    shutil.rmtree(tmp)
+
+
+@exams.command()
+@click.argument('frame_file')
+@click.argument('output_dir')
+@click.option('--variant-list-file', '-f', help='Tab-separated file. '
+              'defining variables for exam variants. Must have a header. '
+              'Header names define variable names, rows define their values. '
+              'ATTENTION: use latin-1 encoding!', required=True)
+@click.option('--name_column', '-n', help='Which column in the variant list '
+              'to use for output file names.', default='STUDENT_EMAIL',
+              show_default=True)
+@click.option('--verbose', '-v', help='Run pdflatex in nonstopmode.',
+              is_flag=True, default=False)
+def compile_variants(frame_file, output_dir, variant_list_file, name_column, verbose=False):
+    """Compile variants of a LaTeX frame file using variables define in a list.
+
+    Arguments:
+
+    - The LaTeX frame file to compile once for each row in the variant list
+      file with the corresponding environment variables set.
+
+    - The output directory in which to place the resulting PDF files.
+
+    \b
+    Use the following snippet in your LaTeX file:
+    %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+    \\usepackage{catchfile}
+    \\newcommand{\\getenv}[2][]{%
+    \\CatchFileEdef{\\temp}{"|kpsewhich --var-value #2"}{\\endlinechar=-1}%
+    \\if\\relax\\detokenize{#1}\\relax\\temp\\else\\let#1\\temp\\fi}
+    \b
+    \\getenv[\\myVar]{MY_VAR}
+    % ...
+    %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+    Now, the macro \\myVar will evaluate to the value in the column with header
+    MY_VAR in your list file in each compile run.
+
+    Repeat the \\getenv line for each variable you want to define in your list
+    file.
+
+    An Example variant-list-file may look like this:
+    
+    \b
+    STUDENT_NAME\tSTUDENT_EMAIL\tMY_VAR
+    Donald Duck\tdonald@example.com\thard-question
+    Mickey Mouse\tmickey@example.com\teasy-question
+    ...
+    """
+    dialect = csv.Sniffer().sniff(
+        next(open(variant_list_file, encoding='latin1')))
+    with open(variant_list_file, encoding='latin1') as f:
+        items = list(csv.DictReader(f, dialect=dialect))
+
+    os.makedirs(output_dir, exist_ok=True)
+    for env in tqdm(items, desc='compiling variants'):
+        compile(frame_file, output_dir, env[name_column], env, verbose)
+
+#!/usr/bin/env python3
+
+import click
+import csv
+import tarfile
+import shutil
+import io
+import os
+import hashlib
+import logging
+
+from dataclasses import dataclass
+
+
+@dataclass
+class Student:
+    name: str
+    email: str
+    salt: str = '42024'
+
+    @property
+    def generated_filename(self):
+        data = f'{self.salt}{self.name}{self.email}'
+        return f'{hashlib.sha1(data.encode()).hexdigest()}.pdf'
+
+
+@dataclass
+class Students(object):
+    list_file: str
+    input_pdf_dir: str
+    output_pdf_dir: str
+    url_prefix: str
+
+    def __post_init__(self):
+        dialect = csv.Sniffer().sniff(
+            next(open(self.list_file, encoding='latin1')))
+        with open(self.list_file, encoding='latin1') as f:
+            reader = csv.DictReader(f, dialect=dialect)
+            self.students = [
+                Student(row['STUDENT_NAME'], row['STUDENT_EMAIL'])
+                for row in reader
+            ]
+        self.resources = {}
+
+    def process_resource(self, ti, buf):
+        item_id = ti.name.split('/')[-2].split('_')[1]
+        item_type = ti.name.split('/')[-1].split('.')[0]
+        if item_type not in ['url', 'module']:
+            return ti, buf
+        if item_id in self.resources:
+            student = self.resources[item_id]
+        elif self.students:
+            student = self.students.pop()
+            fr = os.path.join(self.input_pdf_dir, f'{student.email}.pdf')
+            to = os.path.join(
+                self.output_pdf_dir, f'{student.generated_filename}')
+            logging.info(f'Copying {fr} to {to}')
+            d = os.path.dirname(to)
+            os.makedirs(d, exist_ok=True)
+            open(os.path.join(d, 'index.html'), 'a').close()
+            shutil.copy(fr, to)
+            self.resources[item_id] = student
+        else:
+            return ti, buf
+        buf = buf.read()
+
+        if item_type == 'url':
+            buf = buf.replace(b'STUDENT_NAME', student.name.encode('utf-8'))
+            buf = buf.replace(
+                b'http://STUDENT_URL',
+                os.path.join(self.url_prefix, student.generated_filename).encode('utf-8')
+            )
+        elif item_type == 'module':
+            buf = buf.replace(b'STUDENT_EMAIL', student.email.encode('utf-8'))
+
+        ti.size = len(buf)
+
+        return ti, io.BytesIO(buf)
+
+
+
+@exams.command()
+@click.option('--backup-file', '-b', help='Moodle backup file (.mbz)',
+              required=True)
+@click.option('--student-list', '-s', help='Student list .tsv file.'
+              ' must have column headers, and at least the columns '
+              'STUDENT_NAME and STUDENT_EMAIL. '
+              'ATTENTION: use latin-1 encoding!',
+              required=True)
+@click.option('--input-pdf-dir', '-i', help='Directory with input PDFs.'
+              ' Filenames must be {student_email_address}.pdf',
+              required=True)
+@click.option('--output-pdf-dir', '-o', help='Directory with output PDFs.'
+              ' Will be created if it doesn\'t exist, the contents will be'
+              ' PDFs with hash names plus an empty index.html file',
+              required=True)
+@click.option('--url-prefix', '-u', help='URL Prefix to use inside the '
+              ' Moodle Backup file.', show_default=True,
+              default='https://files.webis.de/teaching/'
+                      'machine-learning-ws20/problem-sheets')
+def insert_moodle(backup_file, student_list,
+         input_pdf_dir, output_pdf_dir,
+         url_prefix):
+    """Update a Moodle backup file with individual students' exam variants.
+
+    \b
+    To generate the backup file:
+    ---------------------------
+    1. Create a new section
+    2. Create a "URL" resource within that section
+        + Name with placeholder STUDENT_NAME
+        + External URL: http://STUDENT_URL
+        + Restrict Access to "User profile field email address is equal to
+          STUDENT_EMAIL"
+        + Add any other settings common to everyone
+    3. Duplicate the file resource as many times as there are students
+    4. Course administration -> Backup
+        + Uncheck everything except "Include activities and resources"
+        + Then, uncheck everything except for the section created under 1. and
+          all the File resources under it
+        + Perform Backup
+    5. Download the generated .mbz file
+    6. Delete the section created under 1. and create a new empty section in
+       its place
+    """
+
+    students = Students(student_list,
+                        input_pdf_dir, output_pdf_dir,
+                        url_prefix)
+
+    output_file = backup_file.replace('.mbz', '_MODIFIED.mbz')
+    with tarfile.open(backup_file) as tf, \
+            tarfile.open(output_file, 'w:gz') as of:
+
+        for item in tf:
+            if not item.isfile():
+                of.addfile(item)
+                continue
+            buf = tf.extractfile(item)
+            if 'activities/url' in item.name:
+                item, buf = students.process_resource(item, buf)
+            of.addfile(item, buf)
+
+        assert len(students.students) == 0, \
+            "There weren't enough resources in the Moodle backup for all " \
+            f" students! left over: {students.students}"
+
+    logging.info(f'Output written to {output_file}')
+