NIFI-7685: Add UTF8 support for FetchFTP

Signed-off-by: Matthew Burgess <[email protected]> This closes apache#4446
davyam · Oct 1, 2020 · 6990f0d · 6990f0d
1 parent e2ccfbb
commit 6990f0d
Show file tree

Hide file tree

Showing 4 changed files with 137 additions and 165 deletions.
diff --git a/.../nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/FetchFTP.java b/.../nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/FetchFTP.java
@@ -74,6 +74,7 @@ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
         properties.add(FTPTransfer.HTTP_PROXY_PASSWORD);
         properties.add(FTPTransfer.BUFFER_SIZE);
         properties.add(FILE_NOT_FOUND_LOG_LEVEL);
+        properties.add(FTPTransfer.UTF8_ENCODING);
         return properties;
     }
 

diff --git a/...andard-processors/src/main/java/org/apache/nifi/processors/standard/util/FTPTransfer.java b/...andard-processors/src/main/java/org/apache/nifi/processors/standard/util/FTPTransfer.java
@@ -38,6 +38,7 @@
 import java.util.function.Supplier;
 import java.util.regex.Pattern;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.apache.commons.net.ftp.FTPClient;
 import org.apache.commons.net.ftp.FTPFile;
 import org.apache.commons.net.ftp.FTPHTTPClient;
@@ -546,19 +547,8 @@ public void sendCommands(final List<String> commands, final FlowFile flowFile) t
         }
     }
 
-    private FTPClient getClient(final FlowFile flowFile) throws IOException {
-        if (client != null) {
-            String desthost = ctx.getProperty(HOSTNAME).evaluateAttributeExpressions(flowFile).getValue();
-            if (remoteHostName.equals(desthost)) {
-                // destination matches so we can keep our current session
-                resetWorkingDirectory();
-                return client;
-            } else {
-                // this flowFile is going to a different destination, reset session
-                close();
-            }
-        }
-
+    @VisibleForTesting
+    protected FTPClient createFTPClient() {
         final ProxyConfiguration proxyConfig = ProxyConfiguration.getConfiguration(ctx, createComponentProxyConfigSupplier(ctx));
 
         final Proxy.Type proxyType = proxyConfig.getProxyType();
@@ -574,6 +564,24 @@ private FTPClient getClient(final FlowFile flowFile) throws IOException {
                 client.setSocketFactory(new SocksProxySocketFactory(new Proxy(proxyType, new InetSocketAddress(proxyHost, proxyPort))));
             }
         }
+
+        return client;
+    }
+
+    private FTPClient getClient(final FlowFile flowFile) throws IOException {
+        if (client != null) {
+            String desthost = ctx.getProperty(HOSTNAME).evaluateAttributeExpressions(flowFile).getValue();
+            if (remoteHostName.equals(desthost)) {
+                // destination matches so we can keep our current session
+                resetWorkingDirectory();
+                return client;
+            } else {
+                // this flowFile is going to a different destination, reset session
+                close();
+            }
+        }
+
+        FTPClient client = createFTPClient();
         this.client = client;
         client.setBufferSize(ctx.getProperty(BUFFER_SIZE).asDataSize(DataUnit.B).intValue());
         client.setDataTimeout(ctx.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());

diff --git a/...e/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestFTP.java b/...e/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestFTP.java
@@ -202,6 +202,33 @@ public void basicFileFetch() throws IOException {
         retrievedFile.assertContentEquals("Just some random test test test chocolate");
     }
 
+    @Test
+    public void basicFileFetchWithUTF8FileName() throws IOException {
+        FileSystem fs = fakeFtpServer.getFileSystem();
+
+        FileEntry sampleFile = new FileEntry("c:\\data\\őűőű.txt");
+        sampleFile.setContents("Just some random test test test chocolate");
+        fs.add(sampleFile);
+
+        TestRunner runner = TestRunners.newTestRunner(FetchFTP.class);
+        runner.setProperty(FetchFTP.HOSTNAME, "localhost");
+        runner.setProperty(FetchFTP.USERNAME, username);
+        runner.setProperty(FTPTransfer.PASSWORD, password);
+        runner.setProperty(FTPTransfer.PORT, String.valueOf(ftpPort));
+        runner.setProperty(FetchFTP.REMOTE_FILENAME, "c:\\data\\őűőű.txt");
+        runner.setProperty(FetchFTP.COMPLETION_STRATEGY, FetchFTP.COMPLETION_MOVE);
+        runner.setProperty(FetchFTP.MOVE_DESTINATION_DIR, "data");
+        runner.setProperty(FTPTransfer.UTF8_ENCODING, "true");
+
+        runner.enqueue("");
+
+        runner.run();
+
+        runner.assertTransferCount(FetchFTP.REL_SUCCESS, 1);
+        final MockFlowFile retrievedFile = runner.getFlowFilesForRelationship(FetchFTP.REL_SUCCESS).get(0);
+        retrievedFile.assertContentEquals("Just some random test test test chocolate");
+    }
+
     @Test
     public void basicFileList() throws IOException, InterruptedException {
         FileSystem results = fakeFtpServer.getFileSystem();