@@ -7,52 +7,52 @@ JSONFILES = release/nime_alt.json release/nime_installations.json release/nime_m
 all: $(BIBFILES) $(CSVFILES) $(YAMLFILES) $(JSONFILES) release/index.html
 
 release/nime_alt.bib:
-	poetry run python nime_bib collate --type alt --format bib --id_order
+	poetry run python nime_bib collate --type alt --format bib
 
 release/nime_alt.csv:
-	poetry run python nime_bib collate --type alt --format csv --id_order
+	poetry run python nime_bib collate --type alt --format csv
 
 release/nime_alt.yaml:
-	poetry run python nime_bib collate --type alt --format yaml --id_order
+	poetry run python nime_bib collate --type alt --format yaml
 
 release/nime_alt.json:
-	poetry run python nime_bib collate --type alt --format json --id_order
+	poetry run python nime_bib collate --type alt --format json
 
 release/nime_installations.bib:
-	poetry run python nime_bib collate --type installation --format bib --id_order
+	poetry run python nime_bib collate --type installation --format bib
 
 release/nime_installations.csv:
-	poetry run python nime_bib collate --type installation --format csv --id_order
+	poetry run python nime_bib collate --type installation --format csv
 
 release/nime_installations.yaml:
-	poetry run python nime_bib collate --type installation --format yaml --id_order
+	poetry run python nime_bib collate --type installation --format yaml
 
 release/nime_installations.json:
-	poetry run python nime_bib collate --type installation --format json --id_order
+	poetry run python nime_bib collate --type installation --format json
 
 release/nime_music.bib:
-	poetry run python nime_bib collate --type music --format bib --id_order
+	poetry run python nime_bib collate --type music --format bib
 
 release/nime_music.csv:
-	poetry run python nime_bib collate --type music --format csv --id_order
+	poetry run python nime_bib collate --type music --format csv
 
 release/nime_music.yaml:
-	poetry run python nime_bib collate --type music --format yaml --id_order
+	poetry run python nime_bib collate --type music --format yaml
 
 release/nime_music.json:
-	poetry run python nime_bib collate --type music --format json --id_order
+	poetry run python nime_bib collate --type music --format json
 
 release/nime_papers.bib:
-	poetry run python nime_bib collate --type paper --format bib --id_order
+	poetry run python nime_bib collate --type paper --format bib
 
 release/nime_papers.csv:
-	poetry run python nime_bib collate --type paper --format csv --id_order
+	poetry run python nime_bib collate --type paper --format csv
 
 release/nime_papers.yaml:
-	poetry run python nime_bib collate --type paper --format yaml --id_order
+	poetry run python nime_bib collate --type paper --format yaml
 
 release/nime_papers.json:
-	poetry run python nime_bib collate --type paper --format json --id_order
+	poetry run python nime_bib collate --type paper --format json
 
 release/index.html:
 	sh scripts/create_release_index.sh
 
@@ -13,15 +13,27 @@
 
 
 def set_id_order(id_order):
-  """Sets the order for the bibtex writer
+  """Sets the entry order for the bibtex writer.
+
+  id_order=True sorts entries by their BibTeX key. Otherwise the writer's
+  default (article number, url, key) is used. Note the trailing comma:
+  ("ID") is just the string "ID", which bibtexparser would read as the two
+  field names "I" and "D" and silently not sort at all.
   """
-  # set ordering property:
-  if (id_order):
-      utils.writer.order_entries_by = ("ID")
+  if id_order:
+      utils.writer.order_entries_by = ("ID",)
       click.secho("ordering by ID")
   else:
       click.secho(f"Using default order: {utils.writer.order_entries_by}")
-  # else, canonical order: #utils.writer.order_entries_by = ("articleno", "url", "ID")
+
+
+def preserve_file_order():
+  """Makes the bibtex writer keep entries in the order they were parsed.
+
+  The source files are maintained in page/article order, and the collated
+  output concatenates them year by year, so this is the canonical order.
+  """
+  utils.writer.order_entries_by = None
 
 
 @click.command()
@@ -73,49 +85,44 @@ def harmonise(year, type, id_order):
     default="paper",
     help="type of proceedings"
 )
-@click.option(
-    "--id_order",
-    "-I",
-    is_flag=True,
-    default=False,
-    help="sorts the output by entry ID/key (default: sort by article/page number)"
-)
 @click.option(
     "--format",
     "-F",
     type=click.Choice(["bib", "csv", "yaml", "json"]),
     default="bib",
     help="format of output"
 )
-def collate(type, id_order, format):
+def collate(type, format):
   """Collates all NIME proceedings of a certain type and saves to an output
   file.
+
+  Entries are output year by year (files sorted by name) and, within a
+  year, in the order they appear in the source file, which is page/article
+  order. The order is the same for every output format and every platform,
+  so two builds of the same sources can be compared byte for byte.
   """
   output_file = utils.collated_path(type, format)
-  bibfiles = []
+  bibfiles = utils.glob_for_proc(type)
   bib_entries = []
-  bib_databases = []
-  for file in utils.glob_for_proc(type):
-      bibfiles.append(file)
 
   with click.progressbar(bibfiles) as bar:
     for bf in bar:
-      with open(bf) as bibtex_file:
+      with open(bf, encoding="utf-8") as bibtex_file:
         bd = bibtexparser.bparser.BibTexParser(
             common_strings=True
         ).parse_file(bibtex_file)
-        bib_databases.append(bd)
         bib_entries.extend(bd.entries)
-  
+
   # set up collated database
+  bd = bibtexparser.bibdatabase.BibDatabase()
   bd.entries = bib_entries
   accent_converter = latex_accents.AccentConverter()
 
   # BibTex Output (preserve latex accents)
   if format == "bib":
-    set_id_order(id_order)
-    with open(output_file, 'w') as bibtex_file:
-        bibtex_file.write(utils.writer.write(bd)) 
+    preserve_file_order()
+    with open(output_file, 'w', encoding="utf-8") as bibtex_file:
+        bibtex_file.write(utils.writer.write(bd))
 
   # Other formats (convert accents to UTF8)
   for e in bib_entries:
@@ -125,70 +132,12 @@ def collate(type, id_order, format):
     bibtex_str = bibtexparser.dumps(small_bd)
     e['bibtex'] = bibtex_str
 
-    # look at entries with \
-    # look at abstract, title, author
-    try: 
-      if '\\' in e['abstract']:
-        e['abstract'] = accent_converter.decode_Tex_Accents(
-            e['abstract'],
-            utf8_or_ascii=1
-        )
-        e['abstract'] = latex_symbols.replace_symbols(e['abstract'])
-        e['abstract'] = accent_converter.decode_Tex_Accents(
-            e['abstract'],
-            utf8_or_ascii=1
-        )  # do this twice to catch them all.
-      if '\\' in e['abstract']:
-        click.secho(
-            f"{e['ID']} abstract still contains backslashes!",
-            fg="red"
-        )
-        click.secho(f"{e['ID']} Abstract: {e['abstract']}", fg="yellow")
-      e['abstract'] = latex_symbols.clean_braces(e['abstract'])
-    except Exception:
-      continue
-
-    try:
-      if '\\' in e['title']:
-        e['title'] = accent_converter.decode_Tex_Accents(
-            e['title'],
-            utf8_or_ascii=1
-        )
-        e['title'] = latex_symbols.replace_symbols(e['title'])
-        e['title'] = accent_converter.decode_Tex_Accents(
-            e['title'],
-            utf8_or_ascii=1
-        )  # do this twice to catch them all.
-      if '\\' in e['title']:
-        click.secho(
-            f"{e['ID']} title still contains backslashes!",
-            fg="red"
-        )
-        click.secho(f"{e['ID']} Title: {e['title']}", fg="yellow")
-      e['title'] = latex_symbols.clean_braces(e['title'])
-    except Exception as exc:
-      click.secho(f"Exception: {exc}", fg="red")
-
-    try:
-      if '\\' in e['author']:
-        e['author'] = accent_converter.decode_Tex_Accents(
-            e['author'],
-            utf8_or_ascii=1
-        )
-        e['author'] = latex_symbols.replace_symbols(e['author'])
-        e['author'] = accent_converter.decode_Tex_Accents(
-            e['author'],
-            utf8_or_ascii=1
-        )  # do this twice to catch them all.
-      if '\\' in e['author']:
-        click.secho(
-            f"{e['ID']} author still contains backslashes!",
-            fg="red"
-        )
-        click.secho(f"{e['ID']} Author: {e['author']}", fg="yellow")
-      e['author'] = latex_symbols.clean_braces(e['author'])
-    except Exception as exc:
-      click.secho(f"Exception: {exc}", fg="red")
+    # Convert LaTeX in the published text fields to UTF-8. Each field is
+    # handled independently: an entry with no abstract must still have its
+    # title and author converted (this used to `continue` on the missing
+    # abstract and skip them, see issue #98).
+    for field in ("abstract", "title", "author"):
+      _convert_entry_field(e, field, accent_converter)
 
     try:
       if '\\' in e['ID']:
@@ -417,7 +366,7 @@ def add_dois(year, csvfile, type, translated):
         # IMPORTANT: In translation mode we do NOT add or modify e['doi'].
 
   # Write back to the bibtex file
-  set_id_order(True)
+  preserve_file_order()
   with open(nime_file, 'w') as bibtex_file:
       bibtex_file.write(utils.writer.write(bib_database))
 
@@ -436,6 +385,23 @@ def _convert_latex(text, accent_converter):
   return text
 
 
+def _convert_entry_field(entry, field, accent_converter):
+  """Converts LaTeX in one text field of an entry to UTF-8, in place.
+
+  Does nothing if the entry lacks the field. Warns in red about backslashes
+  that survive conversion, since those reach the published files as-is.
+  """
+  value = entry.get(field)
+  if value is None:
+    return
+  if '\\' in value:
+    value = _convert_latex(value, accent_converter)
+  if '\\' in value:
+    click.secho(f"{entry['ID']} {field} still contains backslashes!", fg="red")
+    click.secho(f"{entry['ID']} {field}: {value}", fg="yellow")
+  entry[field] = latex_symbols.clean_braces(value)
+
+
 def _raw_entry_keys(text):
   """Returns the BibTeX keys of every entry header found in raw file text.
 
 
@@ -41,18 +41,23 @@ def path_for_proc(year, proc_type):
         return None
 
 def glob_for_proc(proc_type):
-    """Returns the glob of available procs for a given type.
+    """Returns the available proceedings files for a given type, sorted by
+    filename (i.e. by year).
+
+    Sorted deliberately: Path.glob returns filesystem order, which differs
+    between macOS and the Linux CI runner and would make the collated output
+    non-reproducible.
     """
     if proc_type == "paper":
-        return PAPER_PROC.glob(f"nime*.bib")
+        return sorted(PAPER_PROC.glob("nime*.bib"))
     elif proc_type == "installation":
-        return INSTALL_PROC.glob(f"nime*_installations.bib")
+        return sorted(INSTALL_PROC.glob("nime*_installations.bib"))
     elif proc_type == "music":
-        return MUSIC_PROC.glob(f"nime*_music.bib")
+        return sorted(MUSIC_PROC.glob("nime*_music.bib"))
     elif proc_type == "alt":
-        return ALT_PROC.glob(f"nime*_alt.bib")
+        return sorted(ALT_PROC.glob("nime*_alt.bib"))
     else:
-        return None
+        return []
 
 
 # field order for nime proc entries.