diff --git a/collect_app/src/main/java/org/odk/collect/android/activities/FormEntryActivity.java b/collect_app/src/main/java/org/odk/collect/android/activities/FormEntryActivity.java index 5d928ff7653..aa950e5d34d 100644 --- a/collect_app/src/main/java/org/odk/collect/android/activities/FormEntryActivity.java +++ b/collect_app/src/main/java/org/odk/collect/android/activities/FormEntryActivity.java @@ -2603,13 +2603,16 @@ public void loadingComplete(FormLoaderTask task) { } } - // if somehow we end up with a bad language, set it to the default + long start = System.currentTimeMillis(); + Timber.i("calling formController.setLanguage"); try { formController.setLanguage(newLanguage); } catch (Exception e) { + // if somehow we end up with a bad language, set it to the default Timber.e("Ended up with a bad language. %s", newLanguage); formController.setLanguage(defaultLanguage); } + Timber.i("Done in %.3f seconds.", (System.currentTimeMillis() - start) / 1000F); } boolean pendingActivityResult = task.hasPendingActivityResult(); diff --git a/collect_app/src/main/java/org/odk/collect/android/tasks/DownloadFormsTask.java b/collect_app/src/main/java/org/odk/collect/android/tasks/DownloadFormsTask.java index c4e2c4d0f79..e2d545f9a57 100644 --- a/collect_app/src/main/java/org/odk/collect/android/tasks/DownloadFormsTask.java +++ b/collect_app/src/main/java/org/odk/collect/android/tasks/DownloadFormsTask.java @@ -52,6 +52,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.zip.GZIPInputStream; import timber.log.Timber; @@ -93,7 +94,7 @@ protected HashMap doInBackground(ArrayList... Collect.getInstance().getActivityLogger().logAction(this, "downloadForms", String.valueOf(total)); - HashMap result = new HashMap(); + final HashMap result = new HashMap<>(); for (FormDetails fd : toDownload) { publishProgress(fd.formName, String.valueOf(count), String.valueOf(total)); @@ -149,7 +150,15 @@ protected HashMap doInBackground(ArrayList... } try { - checkForBadSubmissionUrl(fileResult); + if (fileResult != null) { + final long start = System.currentTimeMillis(); + Timber.i("Starting an extra parse to check for a bad submission URL. %s", + fileResult.file.getAbsolutePath()); + // todo can we avoid this extra parse? It can run a long time. + checkForBadSubmissionUrl(fileResult); + Timber.i("Parse finished in %.3f seconds.", + (System.currentTimeMillis() - start) / 1000F); + } } catch (IllegalArgumentException e) { message += e.getMessage(); } @@ -272,8 +281,10 @@ private UriResult findExistingOrCreateNewUri(File formFile) throws TaskCancelled v.put(FormsColumns.FORM_MEDIA_PATH, mediaPath); Timber.w("Parsing document %s", formFile.getAbsolutePath()); - - HashMap formInfo = FileUtils.parseXML(formFile); + final long start = System.currentTimeMillis(); + Map formInfo = FileUtils.parseXML(formFile); + Timber.i("Parse finished in %.3f seconds.", + (System.currentTimeMillis() - start) / 1000F); if (isCancelled()) { throw new TaskCancelledException(formFile, "Form " + formFile.getName() diff --git a/collect_app/src/main/java/org/odk/collect/android/tasks/FormLoaderTask.java b/collect_app/src/main/java/org/odk/collect/android/tasks/FormLoaderTask.java index 54e9712cceb..fe21955ebab 100644 --- a/collect_app/src/main/java/org/odk/collect/android/tasks/FormLoaderTask.java +++ b/collect_app/src/main/java/org/odk/collect/android/tasks/FormLoaderTask.java @@ -116,80 +116,31 @@ public FormLoaderTask(String instancePath, String xpath, String waitingXPath) { /** * Initialize {@link FormEntryController} with {@link FormDef} from binary or - * from XML. If given an instance, it will be used to fill the {@link FormDef} - * . + * from XML. If given an instance, it will be used to fill the {@link FormDef}. */ @Override protected FECWrapper doInBackground(String... path) { - FormDef fd = null; - FileInputStream fis = null; errorMsg = null; - String formPath = path[0]; - - File formXml = new File(formPath); - String formHash = FileUtils.getMd5Hash(formXml); - File formBin = new File(Collect.CACHE_PATH + File.separator + formHash + ".formdef"); - - publishProgress( - Collect.getInstance().getString(R.string.survey_loading_reading_form_message)); + final String formPath = path[0]; + final File formXml = new File(formPath); + final FormDef formDef = createFormDefFromCacheOrXml(formPath, formXml); - // FormDef.setDefaultEventNotifier(new EventNotifier() { - // - // @Override - // public void publishEvent(Event event) { - // Log.d("FormDef", event.asLogLine()); - // } - // }); - - if (formBin.exists()) { - // if we have binary, deserialize binary - Timber.i("Attempting to load %s from cached file: %s", - formXml.getName(), formBin.getAbsolutePath()); - fd = deserializeFormDef(formBin); - if (fd == null) { - // some error occured with deserialization. Remove the file, and make a - // new .formdef - // from xml - Timber.w("Deserialization FAILED! Deleting cache file: %s", - formBin.getAbsolutePath()); - formBin.delete(); - } - } - if (fd == null) { - // no binary, read from xml - try { - Timber.i("Attempting to load from: %s", formXml.getAbsolutePath()); - fis = new FileInputStream(formXml); - fd = XFormUtils.getFormFromInputStream(fis); - if (fd == null) { - errorMsg = "Error reading XForm file"; - } else { - serializeFormDef(fd, formPath); - } - } catch (Exception e) { - Timber.e(e); - errorMsg = e.getMessage(); - } finally { - IOUtils.closeQuietly(fis); - } - } - - if (errorMsg != null || fd == null) { + if (errorMsg != null || formDef == null) { return null; } // set paths to /sdcard/odk/forms/formfilename-media/ - String formFileName = formXml.getName().substring(0, formXml.getName().lastIndexOf(".")); - File formMediaDir = new File(formXml.getParent(), formFileName + "-media"); + final String formFileName = formXml.getName().substring(0, formXml.getName().lastIndexOf(".")); + final File formMediaDir = new File(formXml.getParent(), formFileName + "-media"); externalDataManager = new ExternalDataManagerImpl(formMediaDir); // add external data function handlers ExternalDataHandler externalDataHandlerPull = new ExternalDataHandlerPull( externalDataManager); - fd.getEvaluationContext().addFunctionHandler(externalDataHandlerPull); + formDef.getEvaluationContext().addFunctionHandler(externalDataHandlerPull); try { loadExternalData(formMediaDir); @@ -205,51 +156,16 @@ protected FECWrapper doInBackground(String... path) { } // create FormEntryController from formdef - FormEntryModel fem = new FormEntryModel(fd); - FormEntryController fec = new FormEntryController(fem); + final FormEntryModel fem = new FormEntryModel(formDef); + final FormEntryController fec = new FormEntryController(fem); boolean usedSavepoint = false; try { - // import existing data into formdef - if (instancePath != null) { - File instance = new File(instancePath); - File shadowInstance = SaveToDiskTask.savepointFile(instance); - if (shadowInstance.exists() && (shadowInstance.lastModified() - > instance.lastModified())) { - // the savepoint is newer than the saved value of the instance. - // use it. - usedSavepoint = true; - instance = shadowInstance; - Timber.w("Loading instance from shadow file: %s", shadowInstance.getAbsolutePath()); - } - if (instance.exists()) { - // This order is important. Import data, then initialize. - try { - importData(instance, fec); - fd.initialize(false, new InstanceInitializationFactory()); - } catch (RuntimeException e) { - Timber.e(e); - - // SCTO-633 - if (usedSavepoint - && !(e.getCause() instanceof XPathTypeMismatchException)) { - // this means that the .save file is corrupted or 0-sized, so - // don't use it. - usedSavepoint = false; - instancePath = null; - fd.initialize(true, new InstanceInitializationFactory()); - } else { - // this means that the saved instance is corrupted. - throw e; - } - } - } else { - fd.initialize(true, new InstanceInitializationFactory()); - } - } else { - fd.initialize(true, new InstanceInitializationFactory()); - } + Timber.i("Importing existing data"); + final long start = System.currentTimeMillis(); + usedSavepoint = importExistingData(formDef, fec); + Timber.i("Imported in %.3f seconds.", (System.currentTimeMillis() - start) / 1000F); } catch (RuntimeException e) { Timber.e(e); if (e.getCause() instanceof XPathTypeMismatchException) { @@ -270,22 +186,110 @@ protected FECWrapper doInBackground(String... path) { // Remove previous forms ReferenceManager.instance().clearSession(); + processItemSets(formMediaDir); + + // This should get moved to the Application Class + if (ReferenceManager.instance().getFactories().length == 0) { + // this is /sdcard/odk + ReferenceManager.instance().addReferenceFactory(new FileReferenceFactory(Collect.ODK_ROOT)); + } + + // Set jr://... to point to /sdcard/odk/forms/filename-media/ + ReferenceManager.instance().addSessionRootTranslator( + new RootTranslator("jr://images/", "jr://file/forms/" + formFileName + "-media/")); + ReferenceManager.instance().addSessionRootTranslator( + new RootTranslator("jr://image/", "jr://file/forms/" + formFileName + "-media/")); + ReferenceManager.instance().addSessionRootTranslator( + new RootTranslator("jr://audio/", "jr://file/forms/" + formFileName + "-media/")); + ReferenceManager.instance().addSessionRootTranslator( + new RootTranslator("jr://video/", "jr://file/forms/" + formFileName + "-media/")); + + final FormController fc = new FormController(formMediaDir, fec, instancePath == null ? null + : new File(instancePath)); + if (xpath != null) { + // we are resuming after having terminated -- set index to this + // position... + FormIndex idx = fc.getIndexFromXPath(xpath); + fc.jumpToIndex(idx); + } + if (waitingXPath != null) { + FormIndex idx = fc.getIndexFromXPath(waitingXPath); + fc.setIndexWaitingForData(idx); + } + data = new FECWrapper(fc, usedSavepoint); + return data; + } + + private FormDef createFormDefFromCacheOrXml(String formPath, File formXml) { + final String formHash = FileUtils.getMd5Hash(formXml); + + publishProgress( + Collect.getInstance().getString(R.string.survey_loading_reading_form_message)); + + final File cachedForm = new File(Collect.CACHE_PATH + File.separator + formHash + ".formdef"); + if (cachedForm.exists()) { + Timber.i("Attempting to load %s from cached file: %s.", + formXml.getName(), cachedForm.getAbsolutePath()); + final long start = System.currentTimeMillis(); + final FormDef deserializedFormDef = deserializeFormDef(cachedForm); + if (deserializedFormDef != null) { + Timber.i("Loaded in %.3f seconds.", (System.currentTimeMillis() - start) / 1000F); + + return deserializedFormDef; + } + + // An error occurred with deserialization. Remove the file, and make a + // new .formdef from xml. + Timber.w("Deserialization FAILED! Deleting cache file: %s", + cachedForm.getAbsolutePath()); + cachedForm.delete(); + } + + FileInputStream fis = null; + // no binary, read from xml + try { + Timber.i("Attempting to load from: %s", formXml.getAbsolutePath()); + final long start = System.currentTimeMillis(); + fis = new FileInputStream(formXml); + FormDef formDefFromXml = XFormUtils.getFormFromInputStream(fis); + if (formDefFromXml == null) { + errorMsg = "Error reading XForm file"; + } else { + Timber.i("Loaded in %.3f seconds. Now saving to cache.", + (System.currentTimeMillis() - start) / 1000F); + final long start2 = System.currentTimeMillis(); + serializeFormDef(formDefFromXml, formPath); + Timber.i("Saved to cache in %.3f seconds.", + (System.currentTimeMillis() - start2) / 1000F); + return formDefFromXml; + } + } catch (Exception e) { + Timber.e(e); + errorMsg = e.getMessage(); + } finally { + IOUtils.closeQuietly(fis); + } + + return null; + } + + private void processItemSets(File formMediaDir) { // for itemsets.csv, we only check to see if the itemset file has been // updated - File csv = new File(formMediaDir.getAbsolutePath() + "/" + ITEMSETS_CSV); + final File csv = new File(formMediaDir.getAbsolutePath() + "/" + ITEMSETS_CSV); String csvmd5 = null; if (csv.exists()) { csvmd5 = FileUtils.getMd5Hash(csv); boolean readFile = false; - ItemsetDbAdapter ida = new ItemsetDbAdapter(); + final ItemsetDbAdapter ida = new ItemsetDbAdapter(); ida.open(); // get the database entry (if exists) for this itemsets.csv, based // on the path - Cursor c = ida.getItemsets(csv.getAbsolutePath()); + final Cursor c = ida.getItemsets(csv.getAbsolutePath()); if (c != null) { if (c.getCount() == 1) { c.moveToFirst(); // should be only one, ever, if any - String oldmd5 = c.getString(c.getColumnIndex("hash")); + final String oldmd5 = c.getString(c.getColumnIndex("hash")); if (oldmd5.equals(csvmd5)) { // they're equal, do nothing } else { @@ -306,45 +310,50 @@ protected FECWrapper doInBackground(String... path) { readCSV(csv, csvmd5, ItemsetDbAdapter.getMd5FromString(csv.getAbsolutePath())); } } + } - // This should get moved to the Application Class - if (ReferenceManager.instance().getFactories().length == 0) { - // this is /sdcard/odk - ReferenceManager.instance().addReferenceFactory(new FileReferenceFactory(Collect.ODK_ROOT)); - } - - // Set jr://... to point to /sdcard/odk/forms/filename-media/ - ReferenceManager.instance().addSessionRootTranslator( - new RootTranslator("jr://images/", "jr://file/forms/" + formFileName + "-media/")); - ReferenceManager.instance().addSessionRootTranslator( - new RootTranslator("jr://image/", "jr://file/forms/" + formFileName + "-media/")); - ReferenceManager.instance().addSessionRootTranslator( - new RootTranslator("jr://audio/", "jr://file/forms/" + formFileName + "-media/")); - ReferenceManager.instance().addSessionRootTranslator( - new RootTranslator("jr://video/", "jr://file/forms/" + formFileName + "-media/")); - - // clean up vars - fis = null; - fd = null; - formBin = null; - formXml = null; - formPath = null; - - FormController fc = new FormController(formMediaDir, fec, instancePath == null ? null - : new File(instancePath)); - if (xpath != null) { - // we are resuming after having terminated -- set index to this - // position... - FormIndex idx = fc.getIndexFromXPath(xpath); - fc.jumpToIndex(idx); - } - if (waitingXPath != null) { - FormIndex idx = fc.getIndexFromXPath(waitingXPath); - fc.setIndexWaitingForData(idx); + private boolean importExistingData(FormDef formDef, FormEntryController fec) { + final InstanceInitializationFactory instanceInit = new InstanceInitializationFactory(); + boolean usedSavepoint = false; + if (instancePath != null) { + File instance = new File(instancePath); + final File shadowInstance = SaveToDiskTask.savepointFile(instance); + if (shadowInstance.exists() && (shadowInstance.lastModified() + > instance.lastModified())) { + // the savepoint is newer than the saved value of the instance. + // use it. + usedSavepoint = true; + instance = shadowInstance; + Timber.w("Loading instance from shadow file: %s", shadowInstance.getAbsolutePath()); + } + if (instance.exists()) { + // This order is important. Import data, then initialize. + try { + importData(instance, fec); + formDef.initialize(false, instanceInit); + } catch (RuntimeException e) { + Timber.e(e); + + // SCTO-633 + if (usedSavepoint + && !(e.getCause() instanceof XPathTypeMismatchException)) { + // this means that the .save file is corrupted or 0-sized, so + // don't use it. + usedSavepoint = false; + instancePath = null; + formDef.initialize(true, instanceInit); + } else { + // this means that the saved instance is corrupted. + throw e; + } + } + } else { + formDef.initialize(true, instanceInit); + } + } else { + formDef.initialize(true, instanceInit); } - data = new FECWrapper(fc, usedSavepoint); - return data; - + return usedSavepoint; } @SuppressWarnings("unchecked") @@ -412,7 +421,7 @@ protected void onProgressUpdate(String... values) { } } - public boolean importData(File instanceFile, FormEntryController fec) { + private boolean importData(File instanceFile, FormEntryController fec) { publishProgress( Collect.getInstance().getString(R.string.survey_loading_reading_data_message)); diff --git a/collect_app/src/main/java/org/odk/collect/android/utilities/FileUtils.java b/collect_app/src/main/java/org/odk/collect/android/utilities/FileUtils.java index b4ef2c330b7..16f8e9afce6 100644 --- a/collect_app/src/main/java/org/odk/collect/android/utilities/FileUtils.java +++ b/collect_app/src/main/java/org/odk/collect/android/utilities/FileUtils.java @@ -283,8 +283,8 @@ private static String actualCopy(File sourceFile, File destFile) { } public static HashMap parseXML(File xmlFile) { - HashMap fields = new HashMap(); - InputStream is; + final HashMap fields = new HashMap(); + final InputStream is; try { is = new FileInputStream(xmlFile); } catch (FileNotFoundException e1) { @@ -300,78 +300,73 @@ public static HashMap parseXML(File xmlFile) { isr = new InputStreamReader(is); } - if (isr != null) { - - Document doc; + final Document doc; + try { + doc = XFormParser.getXMLDocument(isr); + } catch (IOException e) { + Timber.e(e, "Unable to parse XML document %s", xmlFile.getAbsolutePath()); + throw new IllegalStateException("Unable to parse XML document", e); + } finally { try { - doc = XFormParser.getXMLDocument(isr); + isr.close(); } catch (IOException e) { - Timber.e(e, "Unable to parse XML document %s", xmlFile.getAbsolutePath()); - throw new IllegalStateException("Unable to parse XML document", e); - } finally { - try { - isr.close(); - } catch (IOException e) { - Timber.w("%s error closing from reader", xmlFile.getAbsolutePath()); - } + Timber.w("%s error closing from reader", xmlFile.getAbsolutePath()); } + } - String xforms = "http://www.w3.org/2002/xforms"; - String html = doc.getRootElement().getNamespace(); + final String xforms = "http://www.w3.org/2002/xforms"; + final String html = doc.getRootElement().getNamespace(); - Element head = doc.getRootElement().getElement(html, "head"); - Element title = head.getElement(html, "title"); - if (title != null) { - fields.put(TITLE, XFormParser.getXMLText(title, true)); - } + final Element head = doc.getRootElement().getElement(html, "head"); + final Element title = head.getElement(html, "title"); + if (title != null) { + fields.put(TITLE, XFormParser.getXMLText(title, true)); + } - Element model = getChildElement(head, "model"); - Element cur = getChildElement(model, "instance"); + final Element model = getChildElement(head, "model"); + Element cur = getChildElement(model, "instance"); - int idx = cur.getChildCount(); - int i; - for (i = 0; i < idx; ++i) { - if (cur.isText(i)) { - continue; - } - if (cur.getType(i) == Node.ELEMENT) { - break; - } + final int idx = cur.getChildCount(); + int i; + for (i = 0; i < idx; ++i) { + if (cur.isText(i)) { + continue; } + if (cur.getType(i) == Node.ELEMENT) { + break; + } + } - if (i < idx) { - cur = cur.getElement(i); // this is the first data element - String id = cur.getAttributeValue(null, "id"); - String xmlns = cur.getNamespace(); - - String version = cur.getAttributeValue(null, "version"); - String uiVersion = cur.getAttributeValue(null, "uiVersion"); - if (uiVersion != null) { - // pre-OpenRosa 1.0 variant of spec - Timber.e("Obsolete use of uiVersion -- IGNORED -- only using version: %s", - version); - } + if (i < idx) { + cur = cur.getElement(i); // this is the first data element + final String id = cur.getAttributeValue(null, "id"); - fields.put(FORMID, (id == null) ? xmlns : id); - fields.put(VERSION, (version == null) ? null : version); - } else { - throw new IllegalStateException(xmlFile.getAbsolutePath() + " could not be parsed"); - } - try { - Element submission = model.getElement(xforms, "submission"); - String submissionUri = submission.getAttributeValue(null, "action"); - fields.put(SUBMISSIONURI, submissionUri); - String base64RsaPublicKey = submission.getAttributeValue(null, - "base64RsaPublicKey"); - fields.put(BASE64_RSA_PUBLIC_KEY, - (base64RsaPublicKey == null || base64RsaPublicKey.trim().length() == 0) - ? null : base64RsaPublicKey.trim()); - } catch (Exception e) { - Timber.i("XML file %s does not have a submission element", xmlFile.getAbsolutePath()); - // and that's totally fine. + final String version = cur.getAttributeValue(null, "version"); + final String uiVersion = cur.getAttributeValue(null, "uiVersion"); + if (uiVersion != null) { + // pre-OpenRosa 1.0 variant of spec + Timber.e("Obsolete use of uiVersion -- IGNORED -- only using version: %s", + version); } + fields.put(FORMID, (id == null) ? cur.getNamespace() : id); + fields.put(VERSION, (version == null) ? null : version); + } else { + throw new IllegalStateException(xmlFile.getAbsolutePath() + " could not be parsed"); } + try { + final Element submission = model.getElement(xforms, "submission"); + fields.put(SUBMISSIONURI, submission.getAttributeValue(null, "action")); + final String base64RsaPublicKey = submission.getAttributeValue(null, + "base64RsaPublicKey"); + fields.put(BASE64_RSA_PUBLIC_KEY, + (base64RsaPublicKey == null || base64RsaPublicKey.trim().length() == 0) + ? null : base64RsaPublicKey.trim()); + } catch (Exception e) { + Timber.i("XML file %s does not have a submission element", xmlFile.getAbsolutePath()); + // and that's totally fine. + } + return fields; }