import_helper.php

<?php

/**
 * @file
 * Helper class for data imports.
 *
 * Indicia, the OPAL Online Recording Toolkit.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * any later version.
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see http://www.gnu.org/licenses/gpl.html.
 *
 * @license http://www.gnu.org/licenses/gpl.html GPL 3.0
 * @link https://github.com/indicia-team/warehouse/
 */

/**
 * Link in other required php files.
 */

require_once 'lang.php';
require_once 'helper_base.php';

define('EMBED_REUPLOAD_OFF', 0);
define('EMBED_REUPLOAD_ERRORS_ONLY', 1);
define('EMBED_REUPLOAD_ON', 2);

/**
 * Static helper class that provides methods for dealing with imports.
 */
class import_helper extends helper_base {

  /**
   * Can host system support remembered mappings?
   *
   * Flag set to true if the host system is capable of storing our user's
   * remembered import mappings for future imports.
   *
   * @var bool
   *
   */
  private static $rememberingMappings = TRUE;

  /**
   * @var array List of field to column mappings that we managed to set automatically
   */
  private static $automaticMappings = [];

  /**
   * Outputs an import wizard.
   *
   * The csv file to be imported should be available in the $_POST data.
   * Additionally, if there are any preset values which apply to each row in
   * the import data then you can pass these to the importer in the $_POST
   * data. For example, you could set taxa_taxon_list:taxon_list_id=3 in the
   * $_POST data when importing species data to force it to go into list 3.
   *
   * @param array $options
   *   Options array with the following possibilities:
   *   * **model** - Required. The name of the model data is being imported
   *     into.
   *   * **auth** - Read and write authorisation tokens.
   *   * **presetSettings** - Optional associative array of any preset values
   *     for the import settings. Any settings which have a presetSetting
   *     specified will be ommitted from the settings form.
   *   * **occurrenceAssociations** - set to true to enable import of
   *     associated occurrences or false to disable it. Default false.
   *   * **fieldMap** - array of configurations of the fields available to
   *     import, one per survey. The importer will generate a list of all
   *     possible fields in the database to import into for a given survey.
   *     This typically includes all the standard "core" database fields such
   *     as species name and sample date, as well as a list of all custom
   *     attributes for a survey. This list is quite long and some of the
   *     default core database fields provided might not be appropriate to your
   *     survey dataset, leading to possible confusion. So you can use this
   *     parameter to define database fields and column titles in the
   *     spreadsheet that will automatically map to them. Provide an array,
   *     with each array entry being an associative array containing the
   *     definition of the fields for 1 survey dataset. In the associative
   *     array provide a value called survey_id to link this definition to a
   *     survey dataset. Also provide a value called fields containing a list
   *     of database fields you are defining for this dataset, one per line. If
   *     you want to link this field to a column title then follow the database
   *     field name with an equals, then the column title, e.g.
   *     sample:date=Record date or occAttr:fk_293=Life stage.
   *   * **onlyAllowMappedFields** - set to true and supply field mappings in
   *     the fieldMap parameter to ensure that only the fields you have
   *     specified for the selected survey will be available for selection.
   *     This allows you to hide all the import fields that you don't want to
   *     be used for importing into a given survey dataset, thus tidying up the
   *     list of options to improve ease of use. Default true.
   *   * **skipMappingIfPossible** - set to true to completely bypass the field
   *     to column mappings setup stage of the import tool if all the columns
   *     in the supplied spreadsheet are mapped. Combine this with the fieldMap
   *     parameter to make predefined import configurations that require little
   *     effort to use as long as a matching spreadsheet structure is supplied.
   *   * **embed_reupload** - set to EMBED_REUPLOAD_ON to embed an upload form
   *     into the page shown on completion of an upload, allowing another file
   *     to be uploaded. Or, set to EMBED_REUPLOAD_ERRORS_ONLY to embed the
   *     upload form into this page only when the last upload had errors, in
   *     which case a message is shown explaining that the user can use the
   *     form to upload the errors file. Defaults to EMBED_REUPLOAD_OFF.
   *   * **importPreventCommitBehaviour** - default is 'partial_import' which
   *     allows working rows to commit and reports errors on failing rows.
   *     'user_defined' displays a checkbox allowing the user to select which
   *     behaviour to use and 'prevent' prevents any commit if there are any
   *     errors.
   *   * **importSampleLogic** - @todo Document.
   *   * **allowExcel** - set to TRUE to enable experimental Excel import.
   *     Currently defaults to false but may change to true when the Excel
   *     import development is stable.
   *
   * @return string
   *   HTML for the next page of the importer.
   *
   * @throws \exception
   */

  public static function importer($options) {
    $options = array_merge([
      // Default is to allow working rows to commit and only report errors on
      // failing rows.
      'importPreventCommitBehaviour' => 'partial_import',
      // Default which to not use sample example key for verification.
      'importSampleLogic' => 'consecutive_rows',
      'allowExcel' => TRUE,
    ], $options);
    // Currently the preventCommitsOnError on error option won't work with existing data updates.
    // Hopefully this will change in the future, the problem is the preserve_fields function does
    // not currently preserve the existing data lookup selections because preserve fields was coded first.
    if (!empty($_POST['preventCommitsOnError']) && $_POST['preventCommitsOnError'] == TRUE) {
      self::$javascript .= "
      $('#lookupSelectsample').attr('disabled', 'disabled');
      $('#lookupSelectoccurrence').attr('disabled', 'disabled');
      $('#lookup-mode-warning').show();";
    }
    self::add_resource('jquery_ui');
    self::add_resource('import');
    // If there is no upload total yet and no import step we know to show the
    // very first screen.
    if (!isset($_POST['import_step']) && !isset($_POST['total'])) {
      if (count($_FILES) === 1) {
        return self::importSettingsForm($options);
      }
      else {
        return self::uploadForm($options);
      }
    }
    elseif (isset($_POST['import_step']) && $_POST['import_step'] == 1) {
      // If we have the Prevent Commits On Any Error option on, then the first pass of the upload
      // process will always be to check errors and not commit to DB, so set variable to control this
      if ((isset($_POST['preventCommitsOnError'])&&$_POST['preventCommitsOnError'] == TRUE)||
          (isset($_POST['setting']['preventCommitsOnError'])&&$_POST['setting']['preventCommitsOnError'] == TRUE)) {
        $options['allowCommitToDB'] = FALSE;
      }
      else {
        $options['allowCommitToDB'] = TRUE;
      }
      return self::uploadMappingsForm($options);
    // Import step 2 is only shown if the preventCommitsOnError option has been set.
    // This means we don't commit any rows at all if any errors are found, therefore we need
    // an extra error checking step
    }
    elseif ((isset($_POST['import_step']) && $_POST['import_step'] == 2)) {
      $options['allowCommitToDB'] = FALSE;
      return self::runUpload($options);
    }
    elseif ((isset($_POST['import_step']) && $_POST['import_step'] == 3)) {
      $options['allowCommitToDB'] = TRUE;
      return self::runUpload($options);
    }
    elseif (isset($_POST['total']) && empty($_POST['import_step'])) {
      return self::uploadResult($options);
    }
    else {
      throw new exception('Invalid importer state');
    }
  }

  /**
   * Returns the HTML for a simple file upload form.
   */
  private static function uploadForm($options) {
    $reload = self::get_reload_link_parts();
    $reloadpath = $reload['path'] . '?' . self::array_to_query_string($reload['params']);
    $r = '<form action="' . $reloadpath . '" method="post" enctype="multipart/form-data">';
    $excel = $options['allowExcel'] ? ' or Excel (*.xls, *.xlsx)' : '';
    $r .= '<label for="upload">' . lang::get("Select Comma Separated Values (*.csv)$excel file to upload") . ':</label>';
    $r .= '<input type="file" name="upload" id="upload"/>';
    $r .= '<input type="Submit" value="' . lang::get('Upload') . '"></form>';
    return $r;
  }

  /**
   * Generates the import settings form.
   *
   * If none available, then outputs the upload mappings form.
   *
   * @param array $options
   *   Options array passed to the import control.
   */
  private static function importSettingsForm(array $options) {
    $_SESSION['uploaded_file'] = self::uploadFile($options);
    // By this time, we should always have an existing file.
    if (empty($_SESSION['uploaded_file'])) {
      throw new Exception('File to upload could not be found');
    }

    $request = parent::$base_url . "index.php/services/import/get_import_settings/" . $options['model'];
    $request .= '?' . self::array_to_query_string($options['auth']['read']);
    $switches = isset($options['switches']) && is_array($options['switches']) ? $options['switches'] : [];
    if (!empty($options['occurrenceAssociations'])) {
      $switches['occurrence_associations'] = 't';
    }
    $request .= '&' . self::array_to_query_string($switches);
    $response = self::http_post($request, []);
    if (!empty($response['output'])) {
      // Get the path back to the same page.
      $reload = self::get_reload_link_parts();
      $reloadpath = $reload['path'] . '?' . self::array_to_query_string($reload['params']);
      $r = '<div class="page-notice ui-state-highlight ui-corner-all">' . lang::get('import_settings_instructions') . "</div>\n" .
          "<form method=\"post\" id=\"entry_form\" action=\"$reloadpath\" class=\"iform\">\n" .
          "<fieldset><legend>" . lang::get('Import Settings') . "</legend>\n";
      $formArray = json_decode($response['output'], TRUE);
      if (!is_array($formArray)) {
        if (class_exists('kohana')) {
          kohana::log('error', 'Problem occurred during upload. Sent request to get_import_settings and received invalid response.');
          kohana::log('error', "Request: $request");
          kohana::log('error', 'Response: ' . print_r($response, TRUE));
        }
        return 'Could not upload file. Please check that the indicia_svc_import module is enabled on the Warehouse.';
      }
      $formOptions = array(
        'form' => $formArray,
        'readAuth' => $options['auth']['read'],
        'nocache' => TRUE,
      );
      if (isset($options['presetSettings'])) {
        // Skip parts of the form we have a preset value for.
        $formOptions['extraParams'] = $options['presetSettings'];
      }
      else {
        $formOptions['extraParams'] = [];
      }
      // Copy any $_POST data into the extraParams, as this would mean preset
      // values that are provided by the form which the uploader was triggered
      // from. E.g. if on a species checklist, this could be this checklists ID
      // which the user does not need to pick.
      foreach ($_POST as $key => $value) {
        $formOptions['extraParams'][$key] = $value;
      }
      $form = self::build_params_form($formOptions, $hasVisibleContent);
      // If there are no settings required, skip to the next step.
      if (!$hasVisibleContent) {
        return self::uploadMappingsForm($options);
      }
      $r .= $form;
      if (isset($options['presetSettings'])) {
        // The presets might contain some extra values to apply to every row -
        // must be output as hiddens.
        $extraHiddens = array_diff_key($options['presetSettings'], $formArray);
        unset($extraHiddens['password']);
        foreach ($extraHiddens as $hidden => $value) {
          $r .= "<input type=\"hidden\" name=\"$hidden\" value=\"$value\" />\n";
        }
      }
      // If import behaviour is to be specified by the user, then provide
      // options on the screen for them. If not specified by the user they
      // must be hidden in the background.
      if (($options['importPreventCommitBehaviour'] === 'user_defined') ||
          ($options['importSampleLogic'] === 'user_defined' && ($options['model'] === 'occurrence'||$options['model'] === 'sample'))) {
        $r .= '<hr>';
      }
      // In this case the administrator has specific on the Edit Tab that any
      // errors will prevent the import. Keep this information in a hidden
      // checkbox.
      if ($options['importPreventCommitBehaviour'] === 'prevent') {
        $r .= '<input type="checkbox" style="display:none;" name="preventCommitsOnError" checked>';
      }
      // In this case the administrator has specific on the Edit Tab that any
      // errors will only affect affected rows and working rows can be
      // committed. Keep this information in a hidden checkbox.
      if ($options['importPreventCommitBehaviour'] === 'partial_import') {
        $r .= '<input type="checkbox" style="display:none;" name="preventCommitsOnError" >';
      }
      // Admin has specified that user can provide option so display to screen.
      if ($options['importPreventCommitBehaviour'] === 'user_defined') {
        $r .= data_entry_helper::checkbox(array(
          'label' => lang::get('Reject entire import if there are any errors'),
          'fieldname' => 'preventCommitsOnError',
          'helpText' => 'Select this checkbox to prevent the importing of any rows ' .
              'if there are any errors at all. Leave this checkbox switched off to import valid rows. Please note: Functionality to update ' .
               'existing data is currently disabled when this option is selected, only new data can be imported.'
        ));
      }
      // Same logic for other behaviour option.
      if ($options['importSampleLogic'] === 'sample_ext_key') {
        $r .= '<input type="checkbox" style="display:none;" name="verifySamplesUsingExternalKey" checked>';
      }
      if ($options['importSampleLogic'] === 'consecutive_rows') {
        $r .= '<input type="checkbox" style="display:none;" name="verifySamplesUsingExternalKey" >';
      }
      if ($options['importSampleLogic'] === 'user_defined' && ($options['model'] === 'occurrence' || $options['model'] === 'sample')) {
        $r .= data_entry_helper::checkbox([
          'label' => lang::get('Samples verified by sample key field'),
          'fieldname' => 'verifySamplesUsingExternalKey',
          'helpText' => 'Select this checkbox to verify imported samples using the sample external key field to determine consistency between the imported rows. '.
          'e.g. occurrences with the same external key on the row cannot have different sample dates. Note that rows for the same sample must still be placed consecutively in the import file.'
        ]);
      }
      $r .= '<input type="hidden" name="import_step" value="1" />';
      $r .= '<input type="submit" name="submit" value="' . lang::get('Next') . '" class="ui-corner-all ui-state-default button" />';
      // Copy any $_POST data into the form, as this would mean preset values
      // that are provided by the form which the uploader was triggered from.
      // E.g. if on a species checklist, this could be this checklists ID which
      // the user does not need to pick.
      foreach ($_POST as $key => $value) {
        $r .= "<input type=\"hidden\" name=\"$key\" value=\"$value\" />\n";
      }
      $r .= '</fieldset></form>';
      return $r;
    }
    else {
      // No settings form, so output the mappings form instead which is the
      // next step.
      return self::uploadMappingsForm($options);
    }
  }

  /**
   * Outputs the form for mapping columns to the import fields.
   *
   * @param array $options
   *   Options array passed to the import control.
   */
  private static function uploadMappingsForm(array $options) {
    $t = self::getTranslations([
      'Because you are looking up existing records to import into, required field validation will only be applied when the new data are merged into the existing data during import.',
      'Column in import File',
      'column_mapping_instructions',
      'Maps to attribute',
      'Tasks',
      'The following database attributes must be matched to a column in your import file before you can continue',
      'There are currently two or more drop-downs allocated to the same value.',
      'Used in lookup of existing data?',
    ]);
    self::addLanguageStringsToJs('import', [
      'lookup_selected_help' => '{1} with the same {2} as rows in the import file will be updated.',
      'lookup_unavailable_reason' => 'Looking up {1} based on {2} requires {3} to match but {4} {5} in your import.',
      'not_available_plural' => 'are not available',
      'not_available_singular' => 'is not available',
      'not_imported' => 'Not imported',
      'value_plural' => 'values',
      'value_singular' => 'value',
      'unavailable_lookup_options' => 'Unavailable lookup options',
    ]);
    $filename = basename($_SESSION['uploaded_file']);
    $mappingsAndSettings = self::getMappingsAndSettings($options);
    $settings = $mappingsAndSettings['settings'];
    $request = parent::$base_url . "index.php/services/import/get_import_fields/" . $options['model'];
    $request .= '?' . self::array_to_query_string($options['auth']['read']);
    // Include survey and website information in the request if available, as
    // this limits the availability of custom attributes.
    if (!empty($settings['website_id'])) {
      $request .= '&website_id=' . trim($settings['website_id']);
    }
    if (!empty($settings['survey_id'])) {
      $request .= '&survey_id=' . trim($settings['survey_id']);
    }
    if (!empty($settings['useAssociations']) && $settings['useAssociations']) {
      $request .= '&use_associations=true';
    }
    if (($options['model'] === 'sample' || $options['model'] === 'occurrence')
        && isset($settings['sample:sample_method_id'])
        && trim($settings['sample:sample_method_id']) !== '') {
      $request .= '&sample_method_id=' . trim($settings['sample:sample_method_id']);
    }
    elseif ($options['model'] === 'location'
        && isset($settings['location:location_type_id'])
        && trim($settings['location:location_type_id']) !== '') {
      $request .= '&location_type_id=' . trim($settings['location:location_type_id']);
    }
    elseif ($options['model'] === 'taxa_taxon_list'
        && isset($settings['taxa_taxon_list:taxon_list_id'])
        && trim($settings['taxa_taxon_list:taxon_list_id']) !== '') {
      $request .= '&taxon_list_id=' . trim($settings['taxa_taxon_list:taxon_list_id']);
    }
    $response = self::http_post($request, []);
    $fields = json_decode($response['output'], TRUE);
    if (!is_array($fields)) {
      return "curl request to $request failed. Response " . print_r($response, TRUE);
    }
    // Restrict the fields if there is a setting for this survey Id.
    if (!empty($settings['survey_id'])) {
      self::limitFields($fields, $options, $settings['survey_id']);
    }
    if (isset($options['importMergeFields']) && is_string($options['importMergeFields'])) {
      $options['importMergeFields'] = json_decode($options['importMergeFields']);
    }
    if (isset($options['synonymProcessing']) && is_string($options['synonymProcessing'])) {
      $options['synonymProcessing'] = json_decode($options['synonymProcessing']);
    }
    if (isset($options['importMergeFields']) && $options['importMergeFields'] != '' && $options['importMergeFields'] != '{}') {
      foreach ($options['importMergeFields'] as $modelSpec) {
        if (!isset($modelSpec->model) || ($modelSpec->model = $options['model'])) {
          foreach ($modelSpec->fields as $fieldSpec) {
            foreach ($fieldSpec->virtualFields as $subFieldSpec) {
              // Merge the field in at the end on the similar fields.
              $newFields = [];
              $parts = explode(':', $fieldSpec->fieldName);
              $lastMatch = FALSE;
              foreach ($fields as $key => $value) {
                $keyParts = explode(':', $key);
                if ($lastMatch && $keyParts[0] != $parts[0]) {
                  $lastMatch = FALSE;
                  $newFields[$fieldSpec->fieldName . ':' . $subFieldSpec->fieldNameSuffix] = lang::get($subFieldSpec->description) .
                      ' (' . lang::get('merged to form {1}', lang::get($fieldSpec->description)) . ')';
                }
                elseif (!$lastMatch && $keyParts[0] === $parts[0]) {
                  $lastMatch = TRUE;
                }
                $newFields[$key] = $value;
              }
              if ($lastMatch) {
                $newFields[$fieldSpec->fieldName . ':' . $subFieldSpec->fieldNameSuffix] = lang::get($subFieldSpec->description) .
                    ' (' . lang::get('merged to form {1}', lang::get($fieldSpec->description)) . ')';
              }
              $fields = $newFields;
            }
          }
        }
      }
    }
    if (isset($options['synonymProcessing'])) {
      $synonymProcessing = $options['synonymProcessing'];
      if (isset($synonymProcessing->separateSynonyms) && $synonymProcessing->separateSynonyms === TRUE) {
        $fields['synonym:tracker'] = lang::get("Main record vs Synonym");
        $fields['synonym:identifier'] = lang::get("Field to group records together");
      }
    }
    $request = str_replace('get_import_fields', 'get_required_fields', $request);
    $response = self::http_post($request);
    $responseIds = json_decode($response['output'], TRUE);
    if (!is_array($responseIds)) {
      return "curl request to $request failed. Response " . print_r($response, TRUE);
    }
    $model_required_fields = self::expand_ids_to_fks($responseIds);
    $preset_fields = !empty($settings) ? self::expand_ids_to_fks(array_keys(array_filter($settings))) : [];
    $unlinked_fields = !empty($preset_fields) ? array_diff_key($fields, array_combine($preset_fields, $preset_fields)) : $fields;
    // Only use the required fields that are available for selection - the rest
    // are handled somehow else.
    $unlinked_required_fields = array_intersect($model_required_fields, array_keys($unlinked_fields));
    $columns = self::getImportFileColumns($options);
    $reload = self::get_reload_link_parts();
    $reloadpath = $reload['path'] . '?' . self::array_to_query_string($reload['params']);
    self::clear_website_survey_fields($unlinked_fields, $settings);
    self::clear_website_survey_fields($unlinked_required_fields, $settings);
    $autoFieldMappings = self::getAutoFieldMappings($options, $settings);
    $fieldMap = self::getFieldMap($options, $settings);
    // If the user checked the Remember All checkbox need to remember this
    // setting.
    $checkedRememberAll = isset($autoFieldMappings['rememberall']) ? ' checked="checked"' : '';
    $r = <<<HTML
<form method="post" id="entry_form" action="$reloadpath" class="iform">
  <p>{$t['column_mapping_instructions']}</p>
  <div class="ui-helper-clearfix import-mappings-table">
    <table class="ui-widget ui-widget-content">
      <thead class="ui-widget-header">
        <tr><th>{$t['Column in import File']}</th><th>{$t['Maps to attribute']}</th>
HTML;



    if (self::$rememberingMappings) {
      $r .= "<th>" . lang::get('Remember choice?') .
         "<br/><input type='checkbox' name='RememberAll' id='RememberAll' value='1' title='" .
         lang::get('Tick all boxes to remember every column mapping next time you import.') . "'$checkedRememberAll/></th>";
      self::$javascript .= "
      $('#RememberAll').on('change', function() {
        if (this.checked) {
          $(\".rememberField\").attr(\"checked\",\"checked\")
        } else {
          $(\".rememberField\").removeAttr(\"checked\")
        }
      });\n";
    }

    $request = str_replace('get_required_fields', 'get_existing_record_options', $request);
    $response = self::http_post($request);

    // The $existingDataLookupOptions variable holds the required fields that must be filled in for a particular existing lookup option to work
    // e.g. external_key can't be used unless external_key is filled in
    // However these options come from the importDuplicateCheckCombinations option in the model, some fields from here are not suitable for use
    // and if left in cause problems. Remove this options (held in $importDuplicateCheckCombinationsToRemove)
    $existingDataLookupOptions = [];
    $importDuplicateCheckCombinations = json_decode($response['output'], TRUE);
    if (isset($importDuplicateCheckCombinations[$options['model']])) {
      $importDuplicateCheckCombinationsToRemove = array('taxa_taxon_list:taxon_id', 'sample:sample_method_id');
      foreach ($importDuplicateCheckCombinations[$options['model']] as $idx => $importDuplicateCheckCombination) {
        foreach ($importDuplicateCheckCombination['fields'] as $idx2 => $field) {
          if (!empty($field['fieldName']) && in_array($field['fieldName'], $importDuplicateCheckCombinationsToRemove)) {
            unset($importDuplicateCheckCombinations[$options['model']][$idx]['fields'][$idx2]);
          }
        }
      }
      $existingDataLookupOptions = $importDuplicateCheckCombinations;

      if (!is_array($existingDataLookupOptions)) {
        // There is a possibility that the warehouse is not as advanced as the form: in this case we carry on as if no options are avaailable.
        $existingDataLookupOptions = [];
      }
      if (count($existingDataLookupOptions) > 0) {
        $r .= "<th>{$t['Used in lookup of existing data?']}</th>";
      }
    }

    $r .= '</tr></thead><tbody>';
    $importableColCount = 0;
    foreach ($columns as $column) {
      $column = trim($column ?? '');
      if (!empty($column)) {
        if (!in_array($column, ['Number of problems', 'Problem description', 'Row no.', 'Import ID'])) {
          $importableColCount++;
        }
        $colFieldName = self::columnMachineName($column);
        $r .= "<tr><td>$column</td><td><select name=\"$colFieldName\" id=\"$colFieldName\">";
        $r .= self::getColumnOptions(
          $options['model'],
          $unlinked_fields,
          $column,
          $autoFieldMappings,
          $fieldMap,
          count($existingDataLookupOptions) > 0,
          array_key_exists('allowDataDeletions', $options) ? $options['allowDataDeletions'] : FALSE
        );
        $r .= "</select></td></tr>\n";
      }
    }
    $r .= <<<HTML
      </tbody>
    </table>
    <div id="import-mappings-instructions">
      <h2>$t[Tasks]</h2>
      <div id="required-instructions">
        <span>{$t['The following database attributes must be matched to a column in your import file before you can continue']}</span>
        <ul></ul>
        <br/>
      </div>
      <div id="updating-instructions">
        <span>{$t['Because you are looking up existing records to import into, required field validation will only be applied when the new data are merged into the existing data during import.']}</span>
        <br/>
      </div>
      <div id="duplicate-instructions">
        <span id="duplicate-instruct">{$t['There are currently two or more drop-downs allocated to the same value.']}</span>
        <ul></ul>
        <br/>
      </div>
    </div>
  </div>

HTML;
    $usedOptions = [];
    if (count($existingDataLookupOptions) > 0 || !empty($options['existingRecordLookupMethod'])) {
      $r .= '<div id="lookup-mode-warning" class="alert alert-danger" style="display:none">' . lang::get('Note that updating of existing records is only available when the Prevent Commits On Any Error importer option is not being used').'</div>';
      $r .= '<fieldset><legend>' . lang::get('Lookup of existing records') . '</legend>';
      if (!empty($options['existingRecordLookupMethod'])) {
        // Configuration forces a particular lookup method.
        $msg = lang::get('Data will be used to lookup and update existing records where possible.');
        $r .= <<<HTML
<div class="alert alert-info">$msg</div>

HTML;
      }
      else {
        // User can choose lookup method.
        foreach ($existingDataLookupOptions as $model => $combinations) {
          $r .= '<label for="lookupSelect' . $model . '\">' . lang::get(ucfirst($model) . ' records') . '</label>';
          $r .= "<select name=\"lookupSelect" . $model . "\" id=\"lookupSelect" . $model . "\" class=\"lookupSelects\">";
          $r .= "<option value=\"\" >" . lang::get('Do not look up existing records') . "</option>";
          foreach ($combinations as $combination) {
            if (!in_array($model . $combination['description'], $usedOptions)) {
              array_push($usedOptions, $model . $combination['description']);
              // Each possible field for existing record lookup has the list of
              // fields that need to be filled in for it to work specified as
              // json in its value.
              $optionLabel = self::getLookupOptionLabel($model, $combination['description']);
              $r .= "<option value=\"" . htmlspecialchars(json_encode($combination['fields'])) . "\">" . htmlspecialchars($optionLabel) . "</option>";
            }
          }
          $r .= "</select>";
          $r .= '<div id="lookupHelplookupSelect' . $model . '" class="lookup-selected-help alert alert-info" style="display:none"></div>';
          $r .= '<div><a href="#" id="lookupReasonTogglelookupSelect' . $model . '" class="lookup-reason-toggle" style="display:none">' .
            lang::get('Explain why some options are disabled') . '</a></div>';
          $r .= '<div id="lookupReasonlookupSelect' . $model . '" class="lookup-disabled-reason" style="display:none"></div>';
          $r .= "<br/>";
        }
      }
      $r .= "</fieldset>";
      self::$javascript .= <<<JS
indiciaData.presetFields = [];
indiciaData.enableExistingDataLookup = true;

JS;
      foreach ($settings as $key => $value) {
        if (!is_array($value) && trim((string) $value) !== '') {
          self::$javascript .= "indiciaData.presetFields.push(\"$key\");\n";
        }
      }
    }
    else {
      self::$javascript .= "indiciaData.enableExistingDataLookup = false;\n";
    }
    // We need to rerun this even though we run this earlier in this function.
    // The earlier call wouldn't have retrieved any mappings as
    // get_column_options wouldn't have been run yet.
    $mappingsAndSettings = self::getMappingsAndSettings($options);
    self::send_mappings_and_settings_to_warehouse($filename, $options, $mappingsAndSettings);
    // If skip mapping is on, then we don't actually need to show this page and
    // can skip straight to the upload or error checking stage (which will be
    // determined by runUpload using the allowCommitToDB option).
    if (!empty($options['skipMappingIfPossible']) && $options['skipMappingIfPossible'] == TRUE && count(self::$automaticMappings) === $importableColCount) {
      // Need to pass true to stop the mappings and settings being sent to the warehouse during the runUpload function
      // as we have already done that here
      return self::runUpload($options, TRUE);
    }
    // Preserve the post from the website/survey selection screen.
    if (isset($options['allowCommitToDB'])&&$options['allowCommitToDB'] === FALSE) {
      //If we are error checking before upload we do an extra step, which is import step 2
      $r .= self::preserve_fields($options, $filename, 2, FALSE);
    } else {
      $r .= self::preserve_fields($options, $filename, 3, FALSE);
    }
    $r .= '<input type="submit" name="submit" id="submit" value="' . lang::get('Upload') . '" class="ui-corner-all ui-state-default button" />';
    $r .= '</form>';
    self::$javascript .= "required_fields={};\n";
    foreach ($unlinked_required_fields as $field) {
      $caption = $unlinked_fields[$field];
      if (empty($caption)) {
        $tokens = explode(':', $field);
        $fieldname = $tokens[count($tokens) - 1];
        $caption = lang::get(self::processLabel(preg_replace(array('/^fk_/', '/_id$/'), array('', ''), $fieldname)));
      }
      $caption = self::translate_field($field, $caption, $fieldMap);
      self::$javascript .= "required_fields['$field']='$caption';\n";
    }
    self::$onload_javascript .= <<<JS
// Initial setup.
indiciaFns.detectDuplicateFields();
indiciaFns.updateRequiredFields();
indiciaFns.checkLookupOptions();

JS;
    return $r;
  }

  /**
   * Retrieve the list of column names from the import file.
   *
   * Uses the get_column_names service end-point on the warehouse.
   */
  private static function getImportFileColumns($options) {
    $request = parent::$base_url . "index.php/services/import/get_column_names?file=$_SESSION[uploaded_file]&" . self::array_to_query_string($options['auth']['read']);
    $response = self::http_post($request, []);
    return json_decode($response['output']);
  }

  /* Function used to preserve the post from previous stages as we move through the importer otherwise values are lost from
      2 steps ago. Also preserves the automatic mappings used to skip the mapping stage by saving it to the post */
  private static function preserve_fields($options, $filename, $importStep, $formWrapper) {
    $mappingsAndSettings = self::getMappingsAndSettings($options);
    $settingFields = $mappingsAndSettings['settings'];
    $mappingFields = $mappingsAndSettings['mappings'];
    $reload = self::get_reload_link_parts();
    $reload['params']['uploaded_csv'] = $filename;
    $reloadpath = $reload['path'] . '?' . self::array_to_query_string($reload['params']);
    $r = $formWrapper
      ? "<div><form method=\"post\" id=\"fields_to_retain_form\" action=\"$reloadpath\" class=\"iform\">\n"
      : '';

    foreach ($settingFields as $field => $value) {
      if (!empty($settingFields[$field])) {
        if (!empty($value) && $field !== 'import_step' && $field !== 'submit') {
          $r .= "<input type=\"hidden\" name=\"setting[$field]\" id=\"setting[$field]\" value=\"$value\"/>\n";
        }
      }
    }
    foreach ($mappingFields as $field => $value) {
      if (!empty($mappingFields[$field])) {
        if (is_string($field)&&is_string($value)&&!empty($value) && $field !== 'import_step' && $field !==' submit') {
          $value = htmlspecialchars($value);
          $r .= "<input type=\"hidden\" name=\"mapping[$field]\" id=\"mapping[$field]\" value=\"$value\"/>\n";
        }
      }
    }
    if (!empty($importStep)&&$importStep != NULL) {
      $r .= '<input type="hidden" name="import_step" value="' . $importStep . '" />';
    }
    if ($formWrapper) {
      $r .= '<input id="hidden_submit" type="submit" style="display: none" value="' . lang::get('Upload') . '">';
      $r .= "</form><div>\n";
    }
    return $r;
  }

  /**
   * Converts field mappings configuration text to an array.
   *
   * The fieldMap config is a list of database fields with optional '=column
   * titles' added to them. Need a list of column titles mapped to fields so
   * swap this around.
   */
  private static function extractFieldData($fieldText, &$autoFieldMappings) {
    $fields = self::explode_lines($fieldText);
    foreach ($fields as $field) {
      $tokens = explode('=', $field);
      if (count($tokens) === 2) {
        $autoFieldMappings[self::strForCompare($tokens[1])] = $tokens[0];
      }
    }
  }

  /**
   * Returns an array of field to column title mappings that were previously stored in the user profile,
   * or mappings that were provided via the page's configuration form.
   * If the user profile does not support saving mappings then sets self::$rememberingMappings to false.
   *
   * @param array $options
   *   Options array passed to the import helper which might contain a fieldMap.
   * @param array $settings
   *   Settings array for this import which might contain the survey_id.
   *
   * @return array|mixed
   */
  private static function getAutoFieldMappings($options, $settings) {
    $autoFieldMappings = [];
    // Get the user's checked preference for the import page.
    if (function_exists('hostsite_get_user_field') && function_exists('hostsite_set_user_field')) {
      $json = hostsite_get_user_field('import_field_mappings');
      if ($json === FALSE) {
        if (!hostsite_set_user_field('import_field_mappings', '[]')) {
          self::$rememberingMappings = FALSE;
        }
      }
      else {
        $json = trim($json);
        $autoFieldMappings = json_decode(strtolower(trim($json)), TRUE);
      }
    }
    else {
      // Host does not support user profiles, so we can't remember mappings.
      self::$rememberingMappings = FALSE;
    }
    if (!empty($settings['survey_id']) && !empty($options['fieldMap'])) {
      foreach ($options['fieldMap'] as $surveyFieldMap) {
        if (isset($surveyFieldMap['survey_id']) && isset($surveyFieldMap['fields']) &&
            $surveyFieldMap['survey_id'] == $settings['survey_id']) {
          self::extractFieldData($surveyFieldMap['fields'], $autoFieldMappings);
        }
      }
    }
    elseif (empty($settings['survey_id']) && !empty($options['fieldMap'])) {
      // For locations, there is no survey ID, so do the same but with special
      // survey check.
      foreach ($options['fieldMap'] as $surveyFieldMap) {
        if (!isset($surveyFieldMap['survey_id']) && isset($surveyFieldMap['fields']) /* Used for locations */) {
          self::extractFieldData($surveyFieldMap['fields'], $autoFieldMappings);
        }
      }
    }
    return $autoFieldMappings;
  }

  /**
   * Retrieve any configured field mappings for the chosen survey dataset.
   *
   * @param array $options
   *   Options array passed to the import helper which might contain a fieldMap.
   * @param array $settings
   *   Settings array for this import which might contain the survey_id.
   *
   * @return array
   *   Associative array mapping fields to captions.
   */
  private static function getFieldMap($options, $settings) {
    if (!empty($settings['survey_id']) && !empty($options['fieldMap'])) {
      foreach ($options['fieldMap'] as $surveyFieldMap) {
        if (isset($surveyFieldMap['survey_id']) && isset($surveyFieldMap['fields']) &&
            $surveyFieldMap['survey_id'] == $settings['survey_id']) {
          preg_match_all("/([^=\r\n]+)=([^\r\n]+)/", $surveyFieldMap['fields'], $pairs);
          $pairs[1] = array_map('trim', $pairs[1]);
          $pairs[2] = array_map('trim', $pairs[2]);
          $r = [];
          foreach ($pairs[1] as $idx => $key) {
            if (!isset($r[$key])) {
              $r[$key] = $pairs[2][$idx];
            }
          }
          return $r;
        }
      }
    }
    return [];
  }

  /**
   * If the configuration only allows the supplied fields for a given survey ID, then limits the
   * list of available fields retrieve from the warehouse for this survey to that configured list.
   * @param array $fields Field list obtained from the warehouse for this survey. Disallowed fields
   * will be removed.
   * @param array $options Import helper options array
   * @param integer $survey_id ID of the survey being imported
   */
  private static function limitFields(&$fields, $options, $survey_id) {
    if (isset($options['onlyAllowMappedFields']) && $options['onlyAllowMappedFields'] && isset($options['fieldMap'])) {
      foreach ($options['fieldMap'] as $surveyFieldMap) {
        if (isset($surveyFieldMap['survey_id']) && isset($surveyFieldMap['fields']) &&
            ($surveyFieldMap['survey_id']==$survey_id || $surveyFieldMap['survey_id']=="*" /* Used for locations */)) {
          $allowedFields = self::explode_lines($surveyFieldMap['fields']);
          array_walk($allowedFields, function(&$val) {
            $tokens = explode("=",$val);
            $val = $tokens[0];
          });
          $fields = array_intersect_key($fields, array_combine($allowedFields, $allowedFields));
        }
      }
    }
  }

  /**
   * When an array (e.g. $_POST containing preset import values) has values with actual ids in it, we need to
   * convert these to fk_* so we can compare the array of preset data with other arrays of expected data.
   * @param array $arr Array of IDs.
   */
  private static function expand_ids_to_fks($arr) {
    $ids = preg_grep('/_id$/', $arr);
    foreach ($ids as &$id) {
      $id = preg_replace('/_id$/', '', $id);
      if (strpos($id, ':') === FALSE) {
        $id = "fk_$id";
      }
      else {
        $id = str_replace(':', ':fk_', $id);
      }
    }
    return array_merge($arr, $ids);
  }

  /**
   * Takes an array of fields, and removes the website ID or survey ID fields within the arrays if
   * the website and/or survey id are set in the $settings data.
   * @param array $array Array of fields.
   * @param array $settings Global settings which apply to every row, which may include the website_id
   * and survey_id.
   */
  private static function clear_website_survey_fields(&$array, $settings) {
    foreach ($array as $idx => $field) {
      if (!empty($settings['website_id']) && (preg_match('/:fk_website$/', $idx) || preg_match('/:fk_website$/', $field))) {
        unset($array[$idx]);
      }
      if (!empty($settings['survey_id']) && (preg_match('/:fk_survey$/', $idx) || preg_match('/:fk_survey$/', $field))) {
        unset($array[$idx]);
      }
    }
  }

  /**
   * Display the page which outputs the upload progress bar. Adds JavaScript to the page which performs the chunked upload.
   * @param array $options Array of options passed to the import control.
   * @param boolean $calledFromSkippedMappingsPage Indicates if this function was call by the mappings page if that stage is
	 * being skipped on screen. That function will have already sent the settings and mappings to the warehouse so we don't need
	 * to again.
   */
  private static function runUpload($options, $calledFromSkippedMappingsPage = FALSE) {
    self::add_resource('jquery_ui');
    $filename = $_SESSION['uploaded_file'];
    $reload = self::get_reload_link_parts();
    $reload['params']['uploaded_csv'] = $filename;
    $mappingsAndSettings=self::getMappingsAndSettings($options);
    if ($calledFromSkippedMappingsPage===false) {
      self::send_mappings_and_settings_to_warehouse($filename,$options,$mappingsAndSettings);
    }
    $r = '';
    // If we are using the sample external key to verify samples,
    // then we need to check the sample data is consistant between the
    // rows which share the same external key. If not, warn the user.
    if ($options['model']==='occurrence'||$options['model']==='sample') {
      if (!empty($mappingsAndSettings['settings']['verifySamplesUsingExternalKey'])&&$mappingsAndSettings['settings']['verifySamplesUsingExternalKey']==true) {
        // @TODO convert the following to server side as currently only works for CSV.
        $rows=file($_SESSION['uploaded_file']);
        $checkArrays = self::sample_external_key_issue_checks($options, $rows);
        $inconsistencyFailureRows = $checkArrays['inconsistencyFailureRows'];
        $clusteringFailureRows = $checkArrays['clusteringFailureRows'];
        if (!empty($inconsistencyFailureRows)||!empty($clusteringFailureRows)) {
          $r.= self::display_sample_external_key_data_mismatches($inconsistencyFailureRows,$clusteringFailureRows);
          if (!empty($r))
            return $r;
        }
      }
    }

    if (isset($options['allowCommitToDB'])&&$options['allowCommitToDB']===false) {
      //If we hit this line it means we are doing the error checking step and the next step
      //is step 3 which is the actual upload. Preserve the fields from previous steps in the post
      $r .= self::preserve_fields($options, $filename, 3, TRUE);
    } else {
      //This line is hit if we are doing the actual upload now (rather than error check).
      //The next step is the results step which does not have an import_step number
      $r .= self::preserve_fields($options, $filename, NULL, TRUE);
    }
    // If there is an upload total as this point, it means an error check stage must of just been run, so we
	  // need to check for errors in the response
	  if (isset($_POST['total'])) {
      // If we have reached this line, it means the previous step was the error check stage and we are
      // about to attempt to upload, however we need to skip straight to results if we detected any errors
      $output=self::collect_errors($options,$filename);
      if (!is_array($output) || (isset($output['problems'])&&$output['problems']>0)) {
        return self::displayResultAsErrorCheckStageFailed($options,$output);
      }
      //Need to re-send metadata as we need to call warehouse again for upload (rather than error check)
      $mappingsAndSettings=self::getMappingsAndSettings($options);
      self::send_mappings_and_settings_to_warehouse($filename,$options,$mappingsAndSettings);
    }
    // Progress message depends if we are uploading or simply checking for
    // errors.
    if ($options['allowCommitToDB']) {
      $progressMessage = lang::get('{1} records uploaded');
    } else {
      $progressMessage = lang::get('{1} records checked');
    }
    $errorMessage = 'and {1} error(s) encountered.';
    // initiate local javascript to do the upload with a progress feedback
    $r .= <<<HTML
<div id="progress">
<progress id="progress-bar" class="progress" value="0" max="100">0 %</progress>
HTML;
    if (isset($options['allowCommitToDB']) && $options['allowCommitToDB']) {
      $actionMessage='Preparing to upload.';
    } else {
      $actionMessage='Checking file for errors..';
    }
    $r .= "<div id='progress-text'>$actionMessage.</div>
    </div>
    ";
    self::$onload_javascript .= <<<JS
/**
 * Upload a single chunk of a file, by doing an AJAX get. If there is more, then on receiving the response upload the
 * next chunk.
 */
uploadChunk = function() {
  var limit = 50;
  $.ajax({
    url: indiciaData.warehouseUrl + 'index.php/services/import/upload?offset=' + total + '&limit=' + limit +
         '&filepos=' + filepos + '&uploaded_csv=$filename' +
         '&model=$options[model]&allow_commit_to_db=$options[allowCommitToDB]',
    dataType: 'jsonp',
    crossDomain: true
  })
  .done(function(response) {
    var allowCommitToDB = '$options[allowCommitToDB]';
    var message;
    total = total + response.uploaded;
    filepos = response.filepos;
    message = '$progressMessage'.replace('{1}', total - response.errorCount);
    if (response.errorCount > 0) {
      message += ' $errorMessage'.replace('{1}', response.errorCount);
    }
    jQuery('#progress-text').html(message);
    $('#progress-bar').val(response.progress);
    $('#progress-bar').text(response.progress + ' %');
    if (response.uploaded >= limit) {
      uploadChunk();
    } else {
      if (allowCommitToDB) {
        if (response.errorCount > 0) {
          jQuery('#progress-text').html('Upload finished with errors.');
        } else {
          jQuery('#progress-text').html('Upload complete.');
        }
        //We only need total at end of wizard, so we can just refresh page with total as param to use in the post of next step
      } else {
        jQuery('#progress-text').html('Checks complete.');
      }
      $('#fields_to_retain_form').append('<input type=\"hidden\" name=\"total\" id=\"total\" value=\"'+total+'\"/>');
      $('#fields_to_retain_form').submit();
    }
  })
  .fail(function(r) {
    alert('Error uploading file. More information is in the warehouse logs.');
    jQuery('#progress-text').html('Error uploading file. More information is in the warehouse logs.');
  });
};
var total = 0, filepos = 0;
jQuery('#progress-bar').val(0);
jQuery('#progress-bar').text('0 %');
uploadChunk();

JS;
    return $r;
  }

  /*
   * Collect errors from error checking stage
   */
  private static function collect_errors($options, $filename) {
    $request = parent::$base_url . "index.php/services/import/get_upload_result?uploaded_csv=" . $filename;
    $request .= '&'.self::array_to_query_string($options['auth']['read']);
    $response = self::http_post($request, []);
    if (isset($response['output'])) {
      $output = json_decode($response['output'], TRUE);
    } else {
      $output = [];
    }
    return $output;
  }

  private static function getErrorsMessage($output, $msgLangKey) {
    $downloadInstructions = lang::get($msgLangKey);
    $r = '<p>' . lang::get('{1} problems were detected during the import.', $output['problems']) . ' ' . $downloadInstructions . '</p>';
    $r .= "<div><a class=\"btn btn-primary\" download href=\"$output[file]\">" . lang::get('Download the records that did not import.') . '</a></div>';
    return $r;
  }

  /**
   * Render a summary of import outcomes.
   *
   * @param array $output
   *   Parsed output from get_upload_result service.
   *
   * @return string
   *   HTML summary list.
   */
  private static function getImportSummaryMessage(array $output) {
    $matched = isset($output['matched']) ? (int) $output['matched'] : 0;
    $inserted = isset($output['inserted']) ? (int) $output['inserted'] : 0;
    $lookupDataErrors = isset($output['lookup_data_errors']) ? (int) $output['lookup_data_errors'] : 0;
    $r = '<h3>' . lang::get('Import summary') . '</h3>';
    $r .= '<ul>';
    $r .= '<li>' . lang::get('Existing records matched for update') . ': ' . $matched . '</li>';
    $r .= '<li>' . lang::get('New records inserted') . ': ' . $inserted . '</li>';
    $r .= '<li>' . lang::get('Rows errored due to missing lookup data') . ': ' . $lookupDataErrors . '</li>';
    $r .= '</ul>';
    return $r;
  }

  /*
   * Jump to the results screen if errors have been detected.
   *
   * This only applies if we are preventing all commits if any errors are
   * detected (otherwise uploadResult function is called instead).
   */
  private static function displayResultAsErrorCheckStageFailed($options, $output) {
    // Get the path back to the same page.
    $reload = self::get_reload_link_parts();
    $reloadpath = $reload['path'] . '?' . self::array_to_query_string($reload['params']);
    if (function_exists('hostsite_show_message')) {
      hostsite_show_message(lang::get('Some of the rows in the import file could not be imported.'), 'error');
    }
    $r = self::getErrorsMessage($output, 'no_commits_download_error_file_instructions');
    $r .= "<p>" . lang::get('Once you have finished making corrections ') . "<a href=\"$reloadpath\">" . lang::get('please reupload the file.') . "</a></p>";
    return $r;
  }

  /**
   * Displays the upload result page.
   *
   * @param array $options Array of options passed to the import control.
   */
  private static function uploadResult($options) {
    $options = array_merge([
      'embed_reupload' => EMBED_REUPLOAD_OFF,
    ], $options);
    $request = parent::$base_url . "index.php/services/import/get_upload_result?uploaded_csv=" . $_GET['uploaded_csv'];
    $request .= '&' . self::array_to_query_string($options['auth']['read']);
    $response = self::http_post($request, []);
    $r = '';
    if (isset($response['output'])) {
      $output = json_decode($response['output'], TRUE);
      if (!is_array($output) || !isset($output['problems'])) {
        return lang::get('An error occurred during the upload.') . '<br/>' . print_r($response, TRUE);
      }
      if ($output['problems'] > 0) {
        $class = 'upload-results-errors';
        if (function_exists('hostsite_show_message')) {
          hostsite_show_message(lang::get('Some of the rows in the import file could not be imported.'), 'error');
        }
        $r = self::getErrorsMessage($output, 'partial_commits_download_error_file_instructions');
        $r .= self::getImportSummaryMessage($output);
      }
      else {
        $class = 'upload-results-success';
        if (function_exists('hostsite_show_message')) {
          hostsite_show_message(lang::get('The upload was successful.'));
        }
        else {
          $r = lang::get('The upload was successful.');
        }
        $r .= self::getImportSummaryMessage($output);
      }
    }
    else {
      $r = 'An error occurred during the upload.<br/>' . print_r($response, TRUE);
    }
    $reload = self::get_reload_link_parts();
    unset($reload['params']['total']);
    unset($reload['params']['uploaded_csv']);
    $reloadpath = $reload['path'] . '?' . self::array_to_query_string($reload['params']);

    if (isset($response['output'])) {
      if ((int) $options['embed_reupload'] === EMBED_REUPLOAD_ON && $output['problems'] === 0) {
        $r .= '<p>' . lang::get('If you would like to upload another file you can use the form below.') . '</p>';
        $r .= self::uploadForm($options);
      }
      elseif ((int) $options['embed_reupload'] !== EMBED_REUPLOAD_OFF && $output['problems'] > 0) {
        $r .= '<p>' . lang::get('Once you have downloaded the file containing errors and corrected them you can ' .
          'upload the file again using the form below.') . '</p>';
        $r .= self::uploadForm($options);
      }
      else {
        $r .= '<p>' . lang::get('Would you like to ') . "<a href=\"$reloadpath\">" . lang::get('import another file?') . "</a></p>";
      }
    }
    return "<div class=\"$class\">$r</div>";
  }

  /**
   * Return simplified string for comparisons.
   *
   * Allows less fussy column title matching.
   *
   * @param string $str
   *   String to simplify.
   *
   * @return string
   *   String, lowercased, only alphanumerics.
   */
  private static function strForCompare($str) {
    return preg_replace('/[^\da-z]/', '', strtolower($str));
  }

  private static function remove_utf8_bom($text) {
    $bom = pack('H*','EFBBBF');
    $text = preg_replace("/^$bom/", '', $text);
    return $text;
  }

  /**
   * Return column caption translated to machine name.
   *
   * E.g. consistent way to determine name attr for associated control.
   *
   * @param string $caption
   *   Caption to convert.
   *
   * @return string
   *   Non-alphanumerics replaced with _.
   */
  private static function columnMachineName($caption) {
    // Just in case first column in a CSV file with BOM, remove it.
    $bom = pack('H*', 'EFBBBF');
    $caption = preg_replace("/^$bom/", '', $caption ?? '');
    return preg_replace('/[^A-Za-z0-9]/', '_', trim($caption));
  }

  /**
   * Retrieve column options for import.
   *
   * Returns a list of columns as an list of <options> for inclusion in an HTML
   * drop down, loading the columns from a model that are available to import
   * data into (excluding the id and metadata). Triggers the handling of
   * remembered checkboxes and the associated labelling.
   *
   * This method also attempts to automatically find a match for the columns
   * based on a number of rules and gives the user the chance to save their
   * settings for use the next time they do an import.
   *
   * @param string $model
   *   Name of the model.
   * @param array $fields
   *   List of the available possible import columns.
   * @param string $column
   *   The name of the column from the CSV file currently being worked on.
   * @param array $autoFieldMappings
   *   An array containing the automatic field mappings for the page.
   * @param array $fieldMap
   *   Mappings to captions for this survey dataset.
   * @param bool $includeLookups
   *   Should information on which columns are used for lookup be shown.
   * @param bool $allowDataDeletions
   *   Should the importer allow data to be removed.
   */
  private static function getColumnOptions($model, $fields, $column, $autoFieldMappings, $fieldMap, $includeLookups, $allowDataDeletions = FALSE) {
    $skipped = [
      'image_path', 'created_by_id', 'created_on', 'updated_by_id', 'updated_on',
      'fk_created_by', 'fk_updated_by', 'fk_meaning', 'fk_taxon_meaning',
    ];
    // Also skip deleted column if allow deletions disallowed.
    if (!$allowDataDeletions) {
      $skipped[] = 'deleted';
    }
    // We never want to delete at the term level (needs to be termlists_term).
    unset($fields['term:deleted']);
    // Strip the column of spaces for use in HTML ids.
    $idColumn = str_replace(" ", "", $column);
    $r = '';
    $heading = '';
    $labelListIndex = 0;
    $labelList = [];
    $itWasSaved[$column] = 0;
    foreach ($fields as $field => $caption) {
      if (strpos($field, ":")) {
        list($prefix, $fieldname) = explode(':', $field);
      }
      else {
        $prefix = $model;
        $fieldname = $field;
      }
      // Skip the metadata fields.
      if (!in_array($fieldname, $skipped)) {
        // Make a clean looking caption.
        $caption = self::make_clean_caption($caption, $prefix, $fieldname, $model);
        /*
         * The following creates an array called $labelList which is a list of all captions
         * in the drop-down lists. Using array_count_values the array values are calculated as the number of times
         * each caption occurs for use in duplicate detection.
         * $labelListHeading is an array where the keys are each column we work with concatenated to the heading of the caption we
         * are currently working on.
         */
        $strippedScreenCaption = str_replace(" (from controlled termlist)", "", self::translate_field($field, $caption, $fieldMap));
        $labelList[$labelListIndex] = strtolower($strippedScreenCaption);
        $labelListIndex++;
        if (isset($labelListHeading[$column . $prefix])) {
          $labelListHeading[$column . $prefix] = $labelListHeading[$column . $prefix] . ':' . strtolower($strippedScreenCaption);
        }
        else {
          $labelListHeading[$column . $prefix] = strtolower($strippedScreenCaption);
        }
      }
    }
    $labelList = array_count_values($labelList);
    $multiMatch = [];
    foreach ($fields as $field => $caption) {
      if (strpos($field, ":")) {
        list($prefix, $fieldname) = explode(':', $field);
      }
      else {
        $prefix = $model;
        $fieldname = $field;
      }
      // Make a clean looking default caption. This could be provided by the $fields array, or we have to construct it.
      $defaultCaption = self::make_clean_caption($caption, $prefix, $fieldname, $model);
      // Allow the default caption to be translated or overridden by language files.
      $translatedCaption = self::translate_field($field, $defaultCaption, $fieldMap);
      // Need a version of the caption without "from controlled termlist" as we ignore that for matching.
      $strippedScreenCaption = str_replace(" (from controlled termlist)", "", $translatedCaption);
      $fieldname = preg_replace(['/^fk_/', '/_id$/'], ['', ''], $fieldname);
      unset($option);
      // Skip the metadata fields.
      if (!in_array($fieldname, $skipped)) {
        $selected = FALSE;
        // Get user's saved settings, last parameter is 2 as this forces the
        // system to explode into a maximum of two segments. This means only
        // the first occurrence for the needle is exploded which is desirable
        // in the situation as the field caption contains colons in some
        // situations.
        $colKey = self::strForCompare($column);
        if (!empty($autoFieldMappings[$colKey]) && $autoFieldMappings[$colKey] !== '<not imported>') {
          $savedData = explode(':', $autoFieldMappings[$colKey], 2);
          $savedSectionHeading = $savedData[0];
          $savedMainCaption = $savedData[1];
        }
        else {
          $savedSectionHeading = '';
          $savedMainCaption = '';
        }
        // Detect if the user has saved a column setting that is not 'not
        // imported' then call the method that handles the auto-match rules.
        if (strcasecmp($prefix, $savedSectionHeading) === 0 &&
            strcasecmp($field, $savedSectionHeading . ':' . $savedMainCaption) === 0) {
          $selected = TRUE;
          $itWasSaved[$column] = 1;
          // Even though we have already detected the user has a saved setting,
          // we need to call the auto-detect rules as if it gives the same
          // result then the system acts as if it wasn't saved.
          $saveDetectRulesResult = self::auto_detection_rules($column, $defaultCaption, $strippedScreenCaption, $prefix, $labelList, $itWasSaved[$column], TRUE);
          $itWasSaved[$column] = $saveDetectRulesResult['itWasSaved'];
        }
        else {
          // Only use the auto field selection rules to select the drop-down if
          // there isn't a saved option
          if (!isset($autoFieldMappings[$colKey])) {
            $nonSaveDetectRulesResult = self::auto_detection_rules($column, $defaultCaption, $strippedScreenCaption, $prefix, $labelList, $itWasSaved[$column], FALSE);
            $selected = $nonSaveDetectRulesResult['selected'];
          }
        }
        // As a last resort. If we have a match and find that there is more than
        // one caption with this match, then flag a multiMatch to deal with it
        // later.
        if (
          (
            strcasecmp($strippedScreenCaption, $column) == 0 &&
            $labelList[strtolower($strippedScreenCaption)] > 1
          ) ||
          // There is a special case, due to regex auto detection, where the
          // column, 'location' can ambiguously match either 'Location name' or
          // 'Location (from controlled termlist)'.
          (
            $selected &&
            strcasecmp($column, 'location') == 0 &&
            array_key_exists('location', $labelList) &&
            array_key_exists('location name', $labelList)
          )
        ) {
          $multiMatch[] = $column;
          $optionID = $idColumn . 'Duplicate';
        }
        else {
          $optionID = $idColumn . 'Normal';
        }
        $option = self::model_field_option($field, $defaultCaption, $selected, $optionID, $fieldMap);
        if ($selected) {
          self::$automaticMappings[$column] = $field;
        }
      }

      // If we have got an option for this field, add to the list.
      if (isset($option)) {
        // First check if we need a new heading.
        if ($prefix != $heading) {
          $heading = $prefix;
          $class = '';
          if (isset($labelListHeading[$column . $heading])) {
            $subOptionList = explode(':', $labelListHeading[$column . $heading]);
            $foundDuplicate = FALSE;
            foreach ($subOptionList as $subOption) {
              if (isset($labelList[$subOption]) && $labelList[$subOption] > 1) {
                $class = $idColumn . 'Duplicate';
                $foundDuplicate = TRUE;
              }
              if (isset($labelList[$subOption]) && $labelList[$subOption] == 1 and $foundDuplicate == FALSE)
                $class = $idColumn . 'Normal';
            }
          }
          if (!empty($r))
            $r .= '</optgroup>';
          $r .= "<optgroup class=\"$class\" label=\"";
          $r .= self::processLabel(lang::get($heading)) . '">';
        }
        $r .= $option;
      }
    }
    $r = self::items_to_draw_once_per_import_column($r, $column, $itWasSaved, isset($autoFieldMappings['RememberAll']), $multiMatch, $includeLookups);
    return $r;
  }

  /**
   * This method is used by the mode_field_options method.
   * It has two modes:
   * When $saveDetectedMode is false, the method uses several rules in an attempt to automatically determine
   * a value for one of the csv column drop-downs on the import page.
   * When $saveDetectedMode is true, the method uses the same rules to see if the system would have retrieved the
   * same drop-down value as the one that was saved by the user. If this is the case, the system acts
   * as if the value had been automatically determined rather than saved.
   *
   * @param string $column The CSV column we are currently working with from the import file.
   * @param string $defaultCaption The default, untranslated caption.
   * @param string $strippedScreenCaption A version of an item in the column selection drop-down that has 'from controlled termlist'stripped
   * @param string $prefix Caption prefix
   * each item having a list of regexes to match against
   * @param array $labelList A list of captions and the number of times they occur.
   * @param integer $itWasSaved This is set to 1 if the system detects that the user has a custom saved preference for a csv column drop-down.
   * @param boolean $saveDetectedMode Determines the mode the method is running in
   * @return array Depending on the mode, we either are interested in the $selected value or the $itWasSaved value.
   */
  private static function auto_detection_rules($column, $defaultCaption, $strippedScreenCaption, $prefix, $labelList, $itWasSaved, $saveDetectedMode) {
    $column=trim($column);
    /*
     * This is an array of drop-down options with a list of possible column headings the system will use to match against that option.
     * The key is in the format heading:option, all lowercase e.g. occurrence:comment
     * The value is an array of regexes that the system will automatically match against.
     */
    $alternatives = array(
      "sample:entered sref" => array("/(sample)?((spatial|grid|map)ref(erence)?|lat(\/?)lon(g?))/"),
      "occurrence_2:taxa taxon list (from controlled termlist)" => array("/(2nd|second)(species(latin)?|taxon(latin)?|latin)(name)?/"),
      "occurrence:taxa taxon list (from controlled termlist)" => array("/(species(latin)?|taxon(latin)?|latin)(name)?/"),
      // The following regex was originally /(site|location)(name)?/.
      // It probably should have been bounded by ^ and $.
      // As was, it would match 'location external key', 'location code' and
      // 'location indicia id' which should be reserved for other columns.
      // The backwards compatible fix is kind of ugly :-( .
      "sample:location name" => array("/(site|location(?!code|externalkey|indiciaid))(name)?/"),
      "smpAttr:eunis habitat (from controlled termlist)" => array("/(habitat|eunishabitat)/")
    );
    $selected = FALSE;
    //handle situation where there is a unique exact match
    if (self::strForCompare($strippedScreenCaption) === self::strForCompare($column)
        && $labelList[strtolower($strippedScreenCaption)] == 1) {
      if ($saveDetectedMode) {
        $itWasSaved = 0;
      }
      else {
        $selected = TRUE;
      }
    }
    else {
      //handle the situation where a there isn' a unique match, but there is if you take the heading into account also
      if (strcasecmp($prefix . ' ' . $strippedScreenCaption, $column)==0) {
        if ($saveDetectedMode) {
          $itWasSaved = 0;
        }
        else {
          $selected = TRUE;
        }
      }
      //handle the situation where there is a match with one of the items in the alternatives array.
      if (isset($alternatives[$prefix . ':' . strtolower($defaultCaption)])) {
        foreach ($alternatives[$prefix . ':' . strtolower($defaultCaption)] as $regexp) {
          if (preg_match($regexp, strtolower(str_replace(' ', '', $column)))) {
            if ($saveDetectedMode) {
              $itWasSaved = 0;
            }
            else {
              $selected = TRUE;
            }
            break;
          }
        }
      }
    }
    return array(
      'itWasSaved' => $itWasSaved,
      'selected' => $selected
    );
  }

  /**
   * Used by the getColumnOptions to draw the items that appear once for each of the import columns on the import page.
   * These are the checkboxes, the warning the drop-down setting was saved and also the non-unique match warning
   * @param string $r The HTML to be returned.
   * @param string $column Column from the import CSV file we are currently working with
   * @param integer $itWasSaved This is 1 if a setting is saved for the column and the column would not have been automatically calculated as that value anyway.
   * @param boolean $rememberAll Is the remember all mappings option set?.
   * @param array $multiMatch Array of columns where there are multiple matches for the column and this cannot be resolved.
   * @return string HTMl string
   */
  private static function items_to_draw_once_per_import_column($r, $column, $itWasSaved, $rememberAll, $multiMatch, $includeLookups) {
    $optionID = str_replace(" ", "", $column) . 'Normal';
    $r = "<option value=\"&lt;Not imported&gt;\">&lt;" . lang::get('Not imported') . '&gt;</option>' . $r . '</optgroup>';
    if (self::$rememberingMappings) {
      $inputName = self::columnMachineName($column) . '.Remember';
      $checked = ($itWasSaved[$column] == 1 || $rememberAll) ? ' checked="checked"' : '';
      $r .= <<<TD
<td class="centre">
<input type="checkbox" name="$inputName" class="rememberField" id="$inputName" value="1"$checked
  onclick="if (!this.checked) { $('#RememberAll').removeAttr('checked'); }"
  title="If checked, your selection for this particular column will be saved and automatically selected during future
    imports. Any alterations you make to this default selection in the future will also be remembered until you deselect
    the checkbox.">
</td>
TD;
    }
    if ($includeLookups) {
      $checkboxName = self::columnMachineName($column) . '.Lookup';
      $imgPath = empty(self::$images_path) ? self::relative_client_helper_path() . "../media/images/" : self::$images_path;
      $r .= <<<TD
<td class="centre">
<img style="display: none; width: 16px; height: 16px"
  class="in-lookup"
  src="$imgPath/nuvola/ok-16px.png"
  alt="This field is used to lookup the relevant record in the database.">
</td>
TD;
    }
    if ($itWasSaved[$column] == 1) {
      $r .= "<tr><td></td><td class=\"note\">Please check the suggested mapping above is correct.</td></tr>";
    }
    //If we find there is a match we cannot resolve uniquely, then give the user a checkbox to reduce the drop-down to suggestions only.
    //Do this by hiding items whose class has "Normal" at the end as these are the items that do not contain the duplicates.
    self::$javascript .= "
    showHideClass = function(className,action) {
      var selector = '.'+className;
      if (action==='hide') {
        $(selector).wrap('<span/>');
      } else {
        $(selector).unwrap();
      }
    }\n";
    if (in_array($column, $multiMatch) && $itWasSaved[$column] == 0) {
      $r .= "<tr><td></td><td class=\"note\">There are multiple possible matches for ";
      $r .= "\"$column\"";
      $r .=  "<br/><input type='checkbox' id='$column.OnlyShowMatches' value='1' onclick='
       if (this.checked) {
         showHideClass(\"$optionID\",\"hide\");
       } else {
         showHideClass(\"$optionID\",\"show\");
       }'
      > Only show likely matches in drop-down<br></td></tr>";
    }
    return $r;
  }

  /**
   * Used by the getColumnOptions method to add "from controlled termlist" to the appropriate captions
   * in the drop-downs on the import page.
   * @param string $caption The drop-down item currently being worked on
   * @param string $prefix Caption prefix
   * @param string $fieldname The database field that the caption relates to.
   * @param string $model Name of the model
   * @return string $caption A caption for the column drop-down on the import page.
   */
  private static function make_clean_caption($caption, $prefix, $fieldname, $model) {
    $captionSuffix = (substr($prefix,strlen($prefix)-2,2)=='_2' ? // in a association situation with a second record
                       ' (2)' : '');
    if (empty($caption)) {
      if (substr($fieldname,0,3) === 'fk_') {
        $captionSuffix .= ' (' . lang::get('from controlled termlist') . ')';
      }
      $fieldname = preg_replace(['/^fk_/', '/_id$/'], ['', ''], $fieldname);
      if ($prefix === $model || $prefix=="metaFields" || $prefix==substr($fieldname,0,strlen($prefix))) {
        $caption = self::processLabel($fieldname) . $captionSuffix;
      }
      else {
        $caption = self::processLabel("$fieldname") . $captionSuffix;
      }
    }
    else {
      $caption .= (substr($fieldname,0,3)=='fk_' ? ' (' . lang::get('from controlled termlist') . ')' : '');
    }
    return $caption;
  }

  /**
   * Method to upload the file in the $_FILES array.
   *
   * @param array $options
   *   Options array passed to the import control.
   *
   * @return string
   *   Unique file name allocated to the file.
   */
  private static function uploadFile($options) {
    if (!isset($_POST['import_step'])) {
      // No existing file, but on the first step, so the $_POST data must contain the single file.
      if (count($_FILES)!=1)
        throw new Exception('There must be a single file uploaded to import');
      // reset gets the first array element
      $file = reset($_FILES);
      // Get the original file's extension
      $parts = explode(".",$file['name']);
      $fext = array_pop($parts);
      if (!in_array(strtolower($fext), ['csv', 'xls', 'xlsx'])) {
        throw new Exception('Uploaded file must be a CSV, XLS or XLSX file');
      }
      // Generate a file id to store the upload as
      $destination = time() . rand(0,1000) . "." . $fext;
      $interimPath = self::getInterimImageFolder('fullpath');
      if (move_uploaded_file($file['tmp_name'], "$interimPath$destination")) {
        $r = self::send_file_to_warehouse($destination, false, $options['auth']['write_tokens'], 'import/upload_file', TRUE);
        if ($r === TRUE) {
          return $destination;
        } else {
          throw new Exception($r);
        }
      }
    }
    return '';
  }

  /**
   * Humanize a piece of text by inserting spaces instead of underscores, and making first letter
   * of each phrase a capital.
   *
   * @param string $text The text to alter.
   * @return string The altered string.
   */
  private static function processLabel($text) {
    return ucfirst(preg_replace('/[\s_]+/', ' ', $text));
  }

  /**
   * Returns concise labels for existing-record lookup options.
   *
   * @param string $model
   *   Model name, e.g. location.
   * @param string $description
   *   Option description from importDuplicateCheckCombinations.
   *
   * @return string
   *   Concise option label.
   */
  private static function getLookupOptionLabel($model, $description) {
    $label = lang::get($description);
    $prefix = ucfirst(preg_replace('/_+/', ' ', $model)) . ' ';
    if (stripos($label, $prefix) === 0) {
      $label = substr($label, strlen($prefix));
    }
    return trim($label);
  }

  /**
   * Private method to build a single select option for the model field options.
   * Option is selected if selected=caption (case insensitive).
   * @param string $field Name of the field being output.
   * @param string $caption Caption of the field being output.
   * @param boolean $selected Set to true if outputing the currently selected option.
   * @param string $optionID Id of the current option.
   */
  private static function model_field_option($field, $caption, $selected, $optionID, $fieldMap) {
    $selHtml = ($selected) ? ' selected="selected"' : '';
    $caption = self::translate_field($field, $caption, $fieldMap);
    $r = '<option class=';
    $r .= $optionID;
    $r .= ' value="' . htmlspecialchars($field) . "\"$selHtml>" . htmlspecialchars($caption) . '</option>';
    return $r;
  }

  /**
   * Provides optional translation of field captions by looking for a translation code dd:model:fieldname. If not
   * found returns the original caption.
   *
   * @param string $field
   *   Name of the field being output.
   * @param string $caption
   *   Untranslated caption of the field being output.
   * @param array $fieldMap
   *   Mappings to captions for this survey dataset.
   *
   * @return string
   *   Translated caption.
   */
  private static function translate_field($field, $caption, $fieldMap) {
    if (isset($fieldMap[$field])) {
      return lang::get(ucfirst($fieldMap[$field]));
    }
    else {
      // look in the translation settings to see if this column name needs overriding
      $trans = lang::get("dd:$field");
      // Only update the caption if this actually did anything
      if ($trans != "dd:$field" ) {
        return $trans;
      }
      else {
        return $caption;
      }
    }
  }

  private static function send_mappings_and_settings_to_warehouse($filename, $options, $mappingsAndSettings) {
    $mappings = $mappingsAndSettings['mappings'];
    $settings = $mappingsAndSettings['settings'];
    if (empty($settings['useAssociations']) || !$settings['useAssociations']) {
      $settings = self::remove_unused_associations($options,$settings);
    }
    $settings = self::remove_unused_settings($settings);
    $metadata = self::create_metadata_array($mappings, $settings, $options);

    if (function_exists('hostsite_set_user_field')) {
      self::save_user_import_mappings($mappings);
    }
    $post = array_merge($options['auth']['write_tokens'], $metadata);
    $request = parent::$base_url."index.php/services/import/cache_upload_metadata?uploaded_csv=$filename";
    $response = self::http_post($request, $post);
    if (!isset($response['output']) || $response['output'] != 'OK')
      return "Could not upload the settings metadata. <br/>".print_r($response, true);
    else
      return $mappingsAndSettings;
  }

  /**
   * Clean and sort the mappings.
   *
   * Column mappings must be in the same order as the import file, so sort
   * according to the columns in the file and discard any not in the file.
   *
   * @param array $options
   *   Control options.
   * @param array $mappings
   *   Existing column mappings.
   *
   * @return array
   *   Sorted and cleaned mappings.
   */
  private static function cleanMappings(array $options, array $mappings) {
    if (empty($mappings)) {
      return $mappings;
    }
    $correctedMappings = [];
    // Ensure mappings are clean and in the same order as the column headings.
    $columns = self::getImportFileColumns($options);
    foreach($columns as $column) {
      $internalColumn = self::columnMachineName($column);
      $idx=0;
      // For UTF with BOM, the first heading seems to get underscores attached to the front of it,
      // need to strip these and reinstate the array item in a minute as it doesn't get picked up by the $correctedMappings below
      // (because of the underscores not matching the mappings array).
      foreach ($mappings as $key => $value) {
        if ($idx==0) {
          $trimmedKey=ltrim($key,'_');
          $theValueFromTrimmedKey=$value;
        }
        $idx++;
      }
      if (isset($mappings[$internalColumn])) {
        $correctedMappings[$internalColumn] = $mappings[$internalColumn];
      }
    }
    // Re-instate the first item in the array after stripping BOM underscores (see note above)
    $firstItem = array($trimmedKey => $theValueFromTrimmedKey);
    $correctedMappings = $firstItem + $correctedMappings;
    // Lookup existing record selections can go on the end.
    // @todo Existing record lookup selections should really go in their own metadata section.
    foreach($mappings as $key => $value) {
      if (strpos($key, 'lookupSelect') === 0) {
        $correctedMappings[$key] = $value;
      }
    }
    return $correctedMappings;
  }

  //Collect the mappings and settings from various places depending on importer mode, wizard stage.
  //These can be held in variables, option variable, or the post. Collect as appropriate
  private static function getMappingsAndSettings($options) {
    $mappingsAndSettings = [
      'mappings' => [],
      'settings' => [],
    ];
    // If the last step was skipped because the user did not have any settings to supply, presetSettings contains the presets.
    // Otherwise we'll use the settings form content which already in $_POST so will overwrite presetSettings.
    if (isset($options['presetSettings'])) {
      $mappingsAndSettings['settings'] = $options['presetSettings'];
    }
    // Collect settings from a designated array in the post if available.
    if (!empty($_POST['setting'])) {
      // If the post does not contain a specific array for settings, then just
      // get the settings as the general post fields.
      $mappingsAndSettings['settings'] = array_merge($mappingsAndSettings['settings'], $_POST['setting']);
    }
    // Import step 1 is the settings form (data to apply to every row).
    if (isset($_POST['import_step']) && $_POST['import_step'] === '1') {
      $mappingsAndSettings['settings'] = array_merge($mappingsAndSettings['settings'], $_POST);
    }
    // The settings should simply be the settings, so remove any mappings or
    // settings sub-arrays if these have become jumbled up inside our settings
    // array.
    if (!empty($mappingsAndSettings['settings']['mapping'])) {
      unset($mappingsAndSettings['settings']['mapping']);
    }
    if (!empty($mappingsAndSettings['settings']['setting'])) {
      unset($mappingsAndSettings['settings']['setting']);
    }
    // If we are skipping the mappings page, then the mapping will be in the
    // automatic mappings variable. Change the keys in this array so that
    // spaces are replaced with underscores so the mappings are the same as if
    // they had been stored in the post.
    if (!empty(self::$automaticMappings) && !empty($options['skipMappingIfPossible']) && $options['skipMappingIfPossible'] == TRUE) {
      $adjustedAutomaticMappings = [];
      foreach (self::$automaticMappings as $key => $automaticMap) {
        $adjustedAutomaticMappings[self::columnMachineName($key)] = $automaticMap;
      }
      $mappingsAndSettings['mappings'] = $adjustedAutomaticMappings;
    }
    // Import step 2 or 3 are both mappings forms.
    //so we can cleanup any remaining fields in the post as they will be mappings not settings
    if (isset($_POST['import_step']) && in_array($_POST['import_step'], ['2', '3'])) {
      $mappingsAndSettings['mappings'] = array_merge($mappingsAndSettings['mappings'], $_POST);
    }
    // If a configured existing record lookup method, copy it over.
    if (!empty($options['existingRecordLookupMethod'])) {
      $mappingsAndSettings['mappings']["lookupSelect$options[model]"] = $options['existingRecordLookupMethod'];
    }
    $mappingsAndSettings['mappings'] = self::cleanMappings($options, $mappingsAndSettings['mappings']);
    // The mappings should simply be the mappings, so remove any mappings or
    // settings sub-arrays if these have become jumbled up inside our mappings
    // array.
    if (!empty($mappingsAndSettings['mappings']['mapping'])) {
      unset($mappingsAndSettings['mappings']['mapping']);
    }
    if (!empty($mappingsAndSettings['mappings']['setting'])) {
      unset($mappingsAndSettings['mappings']['setting']);
    }
    return $mappingsAndSettings;
  }

  // when not using associations make sure that the association fields are not passed through.
  // These fields would confuse the association detection logic.
  private static function remove_unused_associations($options,$settings) {
    foreach($settings as $key => $value) {
        $parts = explode(':', $key);
        if($parts[0]==$options['model'].'_association' || $parts[0]==$options['model'].'_2')
          unset($settings[$key]);
    }
    return $settings;
  }

  // only want defaults that actually have a value - others can be set on a per-row basis by mapping to a column
  private static function remove_unused_settings($settings) {
    foreach ($settings as $key => $value) {
      if (empty($value)) {
        unset($settings[$key]);
      }
    }
    return $settings;
  }

  private static function create_metadata_array($mappings, $settings, $options) {
    $metadata = [];
    if (function_exists('hostsite_get_user_field')) {
      $metadata['user_id'] = hostsite_get_user_field('indicia_user_id');
    }
    if (!empty($mappings)) {
      $metadata['mappings'] = json_encode($mappings);
    }
    if (!empty($settings)) {
      $metadata['settings'] = json_encode($settings);
    }
    if (!empty($options['importMergeFields'])) {
      $metadata['importMergeFields'] = json_encode($options['importMergeFields']);
    }
    if (!empty($options['synonymProcessing'])) {
      $metadata['synonymProcessing'] = json_encode($options['synonymProcessing']);
    }
    return $metadata;
  }

  private static function save_user_import_mappings($mappings) {
    $userSettings = [];
    foreach ($mappings as $column => $setting) {
      $userSettings[str_replace("_", " ", $column)] = $setting;
    }
    //if the user has not selected the Remember checkbox for a column setting and the Remember All checkbox is not selected
    //then forget the user's saved setting for that column.
    foreach ($userSettings as $column => $setting) {
      if (!isset($userSettings[$column.' '.'Remember']) && $column!='RememberAll')
        unset($userSettings[$column]);
    }
    hostsite_set_user_field("import_field_mappings", json_encode($userSettings));
  }

  /*
   * If we are using the sample external key as the indicator of which samples
   * the occurrences go into, then we need to check the sample data is consistant between the
   * rows which share the same external key, if not, warn the user.
   * We also need to check that rows with the same sample external key appear on consecutive
   * rows (otherwise the importer would create separate samples)
   */
  private static function sample_external_key_issue_checks($options, $rows) {
    $mappingsAndSettings = self::getMappingsAndSettings($options);
    $columnIdx=0;
    $columnIdxsToCheck=[];
    // Cycle through each of the column mappings and get the position of the sample external key column
    foreach ($mappingsAndSettings['mappings'] as $columnName=>$mapping) {
      if ($mapping==='sample:external_key') {
        $sampleKeyIdx=$columnIdx;
      }
      if ((substr($mapping,0,7) === 'sample:'||substr($mapping,0,8) === 'smpAttr:')&&$mapping!=='sample:external_key') {
        array_push($columnIdxsToCheck,$columnIdx);
      }
      $columnIdx++;
    }
    // Hold the latest row which has a given sample external key. All rows with matching external keys must have consistant
	  // sample data, so we only need to hold one for examination
	  $latestRowForEachSampleKey=[];
    // Rows which have inconsistencies
    $inconsistencyFailureRows=[];
    $clusteringFailureRows=[];
    // Flag individual rows which have the same sample external key but the sample data such as the date is inconsistent
    $rowInconsistencyFailure=false;
    // Flag individual rows which have the same sample external key but are not on consecutive rows as this would cause
    // two separate samples which would not be intended
    $rowClusteringFailure=false;
    $rowNumber=0;
    foreach ($rows as $rowNum=>$fileRow) {
      $fileRow=trim($fileRow);
      // Explode individual columns
      $rowArray=explode(',',$fileRow);
      // If the sample key isn't empty on the row, continue to work on the row
      if (!empty($sampleKeyIdx)&&!empty($rowArray[$sampleKeyIdx])) {
        // If the row we are working on has the same sample external key as one of the previous rows then
        // continue
        if (array_key_exists($rowArray[$sampleKeyIdx],$latestRowForEachSampleKey)) {
          // Cycle through each colum on the row
          foreach ($rowArray as $dataCellIdx=>$dataCell) {
            // If any of the row sample columns mismatches an earlier row that has same external key, then flag failure
            if (in_array($dataCellIdx,$columnIdxsToCheck)&&$dataCell!==$latestRowForEachSampleKey[$rowArray[$sampleKeyIdx]][$dataCellIdx]) {
              $rowInconsistencyFailure=true;
            }
            // If the current row number minus the row number of the last row with the same sample external key is bigger
            // than 1 then we know the rows are not consecutive so we can flag a clustering failure to warn the user about
            if ((integer)$rowNumber-(integer)$latestRowForEachSampleKey[$rowArray[$sampleKeyIdx]]['row_number']>1) {
              $rowClusteringFailure=true;
            }
          }
        }
        // Ignore header row
	      // Save the most recent row for each Sample External Key
        if ($rowNum!==0) {
          $latestRowForEachSampleKey[$rowArray[$sampleKeyIdx]]=$rowArray;
          $latestRowForEachSampleKey[$rowArray[$sampleKeyIdx]]['row_number']=$rowNumber;
        }
	      // Flag rows with the same sample external key but different sample data such as dates
	      if ($rowInconsistencyFailure===true) {
	        $inconsistencyFailureRows[$rowNumber]=$fileRow;
        }
        // Flag rows with same sample external key which are not on consecutive rows
	      if ($rowClusteringFailure===true) {
	        $clusteringFailureRows[$rowNumber]=$fileRow;
	      }
	      $rowNumber++;
      }
    }
    $returnArray=[];
    $returnArray['inconsistencyFailureRows']=$inconsistencyFailureRows;
    $returnArray['clusteringFailureRows']=$clusteringFailureRows;
    return $returnArray;
  }

  /*
   * Show results of any consistency issues between rows with the same sample external key
   * (if using that import mode)
   */
  private static function display_sample_external_key_data_mismatches($inconsistencyFailureRows=[],$clusteringFailureRows=[]) {
    $r='';
    $r.='<div><p>You have selected to use the Sample External Key to determine which samples '
        . 'your occurrences are placed into. A scan has been made of your data and problems have been found. '
        . '<em>Note that the listed row numbers exclude the header row.</em></p></div>';
    if (!empty($inconsistencyFailureRows)) {
      $r.='<div><p><b>Inconsistancies have been found in the sample data on your rows which '
          . 'have a matching external key. Please correct your original file and select the '
          . 're-upload option.</b></p><p>The following rows have been found to have inconsistancies:</p></div>';
      foreach ($inconsistencyFailureRows as $rowNum=>$inconsistencyFailureRow) {
        $r.= '<em>'.$inconsistencyFailureRow.' (row number '.$rowNum.')</em><br>';
      }
      $r .= '<div><p>A row is considered to be inconsistant if the sample key matches an earlier row but some of the sample '
          . 'data (such as the date) is different.</p></div>';
    }
    if (!empty($clusteringFailureRows)) {
      $r.='<div><p><b>The following rows have been found to have matching sample external keys but are separated inside the import file. '
          . 'Rows which you need to go into the same sample should be placed on consecutive rows.</b></p></div>';
      foreach ($clusteringFailureRows as $rowNum=>$clusteringFailureRow) {
        $r.= '<em>'.$clusteringFailureRow.' (row number '.$rowNum.')</em><br>';
      }
    }
    $reload = self::get_reload_link_parts();
    unset($reload['params']['total']);
    unset($reload['params']['uploaded_csv']);
    $reloadpath = $reload['path'] . '?' . self::array_to_query_string($reload['params']);
    $r .= "<p>".lang::get('Once you have finished making corrections to the original file ')."<a href=\"$reloadpath\">".lang::get('please reupload the file.')."</a></p>";
    return $r;
  }
}