[SPARK-45232][SQL][DOCS] Add missing function groups to SQL references

### What changes were proposed in this pull request? Add missing function groups to SQL references: - xml_funcs - lambda_funcs - collection_funcs - url_funcs - hash_funcs - struct_funcs Note that this PR doesn't fix `table_funcs`: 1, `gen-sql-functions-docs.py` doesn't work properly with `TableFunctionRegistry`, I took a cursory look but fail to fix it; 2, table functions except `range` (e.g. `explode`) were already contained in `Generator Functions`, not sure we need to show them twice. ### Why are the changes needed? when referring to the SQL references, I find many functions are missing https://spark.apache.org/docs/latest/sql-ref-functions.html. ### Does this PR introduce _any_ user-facing change? yes ### How was this patch tested? manually check ### Was this patch authored or co-authored using generative AI tooling? no Closes apache#43011 from zhengruifeng/doc_xml_functions. Authored-by: Ruifeng Zheng <[email protected]> Signed-off-by: Ruifeng Zheng <[email protected]>
wrschneider · Sep 26, 2023 · 9a44dc4 · 9a44dc4
1 parent 91b7607
commit 9a44dc4
Show file tree

Hide file tree

Showing 3 changed files with 86 additions and 23 deletions.
diff --git a/docs/sql-ref-functions-builtin.md b/docs/sql-ref-functions-builtin.md
@@ -47,6 +47,26 @@ license: |
     {% endif %}
 {% endfor %}
 
+{% for static_file in site.static_files %}
+{% if static_file.name == 'generated-collection-funcs-table.html' %}
+### Collection Functions
+{% include_relative generated-collection-funcs-table.html %}
+#### Examples
+{% include_relative generated-collection-funcs-examples.html %}
+{% break %}
+{% endif %}
+{% endfor %}
+
+{% for static_file in site.static_files %}
+{% if static_file.name == 'generated-struct-funcs-table.html' %}
+### STRUCT Functions
+{% include_relative generated-struct-funcs-table.html %}
+#### Examples
+{% include_relative generated-struct-funcs-examples.html %}
+{% break %}
+{% endif %}
+{% endfor %}
+
 {% for static_file in site.static_files %}
     {% if static_file.name == 'generated-map-funcs-table.html' %}
 ### Map Functions
@@ -67,16 +87,6 @@ license: |
     {% endif %}
 {% endfor %}
 
-{% for static_file in site.static_files %}
-    {% if static_file.name == 'generated-json-funcs-table.html' %}
-### JSON Functions
-{% include_relative generated-json-funcs-table.html %}
-#### Examples
-{% include_relative generated-json-funcs-examples.html %}
-        {% break %}
-    {% endif %}
-{% endfor %}
-
 {% for static_file in site.static_files %}
     {% if static_file.name == 'generated-math-funcs-table.html' %}
 ### Mathematical Functions
@@ -107,6 +117,56 @@ license: |
     {% endif %}
 {% endfor %}
 
+{% for static_file in site.static_files %}
+{% if static_file.name == 'generated-hash-funcs-table.html' %}
+### Hash Functions
+{% include_relative generated-hash-funcs-table.html %}
+#### Examples
+{% include_relative generated-hash-funcs-examples.html %}
+{% break %}
+{% endif %}
+{% endfor %}
+
+{% for static_file in site.static_files %}
+{% if static_file.name == 'generated-csv-funcs-table.html' %}
+### CSV Functions
+{% include_relative generated-csv-funcs-table.html %}
+#### Examples
+{% include_relative generated-csv-funcs-examples.html %}
+{% break %}
+{% endif %}
+{% endfor %}
+
+{% for static_file in site.static_files %}
+{% if static_file.name == 'generated-json-funcs-table.html' %}
+### JSON Functions
+{% include_relative generated-json-funcs-table.html %}
+#### Examples
+{% include_relative generated-json-funcs-examples.html %}
+{% break %}
+{% endif %}
+{% endfor %}
+
+{% for static_file in site.static_files %}
+{% if static_file.name == 'generated-xml-funcs-table.html' %}
+### XML Functions
+{% include_relative generated-xml-funcs-table.html %}
+#### Examples
+{% include_relative generated-xml-funcs-examples.html %}
+{% break %}
+{% endif %}
+{% endfor %}
+
+{% for static_file in site.static_files %}
+{% if static_file.name == 'generated-url-funcs-table.html' %}
+### URL Functions
+{% include_relative generated-url-funcs-table.html %}
+#### Examples
+{% include_relative generated-url-funcs-examples.html %}
+{% break %}
+{% endif %}
+{% endfor %}
+
 {% for static_file in site.static_files %}
     {% if static_file.name == 'generated-bitwise-funcs-table.html' %}
 ### Bitwise Functions
@@ -137,16 +197,6 @@ license: |
     {% endif %}
 {% endfor %}
 
-{% for static_file in site.static_files %}
-    {% if static_file.name == 'generated-csv-funcs-table.html' %}
-### Csv Functions
-{% include_relative generated-csv-funcs-table.html %}
-#### Examples
-{% include_relative generated-csv-funcs-examples.html %}
-        {% break %}
-    {% endif %}
-{% endfor %}
-
 {% for static_file in site.static_files %}
     {% if static_file.name == 'generated-misc-funcs-table.html' %}
 ### Misc Functions

diff --git a/docs/sql-ref-functions.md b/docs/sql-ref-functions.md
@@ -29,16 +29,21 @@ This subsection presents the usages and descriptions of these functions.
 
 #### Scalar Functions
  * [Array Functions](sql-ref-functions-builtin.html#array-functions)
+ * [Collection Functions](sql-ref-functions-builtin.html#collection-functions)
+ * [Struct Functions](sql-ref-functions-builtin.html#struct-functions)
  * [Map Functions](sql-ref-functions-builtin.html#map-functions)
  * [Date and Timestamp Functions](sql-ref-functions-builtin.html#date-and-timestamp-functions)
- * [JSON Functions](sql-ref-functions-builtin.html#json-functions)
  * [Mathematical Functions](sql-ref-functions-builtin.html#mathematical-functions)
  * [String Functions](sql-ref-functions-builtin.html#string-functions)
  * [Bitwise Functions](sql-ref-functions-builtin.html#bitwise-functions)
  * [Conversion Functions](sql-ref-functions-builtin.html#conversion-functions)
  * [Conditional Functions](sql-ref-functions-builtin.html#conditional-functions)
  * [Predicate Functions](sql-ref-functions-builtin.html#predicate-functions)
- * [Csv Functions](sql-ref-functions-builtin.html#csv-functions)
+ * [Hash Functions](sql-ref-functions-builtin.html#hash-functions)
+ * [CSV Functions](sql-ref-functions-builtin.html#csv-functions)
+ * [JSON Functions](sql-ref-functions-builtin.html#json-functions)
+ * [XML Functions](sql-ref-functions-builtin.html#xml-functions)
+ * [URL Functions](sql-ref-functions-builtin.html#url-functions)
  * [Misc Functions](sql-ref-functions-builtin.html#misc-functions)
 
 #### Aggregate-like Functions

diff --git a/sql/gen-sql-functions-docs.py b/sql/gen-sql-functions-docs.py
@@ -34,6 +34,8 @@
     "math_funcs", "conditional_funcs", "generator_funcs",
     "predicate_funcs", "string_funcs", "misc_funcs",
     "bitwise_funcs", "conversion_funcs", "csv_funcs",
+    "xml_funcs", "lambda_funcs", "collection_funcs",
+    "url_funcs", "hash_funcs", "struct_funcs",
 }
 
 
@@ -50,13 +52,19 @@ def _list_grouped_function_infos(jvm):
         name = jinfo.getName()
         if (name == "raise_error"):
             continue
+
+        # SPARK-45232: convert lambda_funcs to collection_funcs in doc generation
+        group = jinfo.getGroup()
+        if group == "lambda_funcs":
+            group = "collection_funcs"
+
         usage = jinfo.getUsage()
         usage = usage.replace("_FUNC_", name) if usage is not None else usage
         infos.append(ExpressionInfo(
             name=name,
             usage=usage,
             examples=jinfo.getExamples().replace("_FUNC_", name),
-            group=jinfo.getGroup()))
+            group=group))
 
     # Groups expression info by each group value
     grouped_infos = itertools.groupby(sorted(infos, key=lambda x: x.group), key=lambda x: x.group)