diff --git a/.circleci/config.yml b/.circleci/config.yml index 701bd51..7314f1c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -50,7 +50,7 @@ jobs: cd onnx-mlir/build cmake --build . --target onnx-mlir-doc # Check whether dialect documentation is up-to-date. - diff doc/Dialects ../doc/Dialects + diff docs/Dialects ../docs/Dialects - run: name: Print the Current Time command: date diff --git a/CMakeLists.txt b/CMakeLists.txt index 4752d4d..f2f7dee 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,5 +27,5 @@ add_subdirectory(third_party/variant) set(CMAKE_CXX_STANDARD 14) add_subdirectory(utils) add_subdirectory(src) -add_subdirectory(doc) +add_subdirectory(docs) add_subdirectory(test) \ No newline at end of file diff --git a/MLIR.cmake b/MLIR.cmake index aede48f..17b9fee 100644 --- a/MLIR.cmake +++ b/MLIR.cmake @@ -238,7 +238,7 @@ function(add_onnx_mlir_dialect_doc dialect dialect_tablegen_file) # Generate Dialect Documentation set(LLVM_TARGET_DEFINITIONS ${dialect_tablegen_file}) onnx_mlir_tablegen(${dialect}.md -gen-op-doc) - set(GEN_DOC_FILE ${ONNX_MLIR_BIN_ROOT}/doc/Dialects/${dialect}.md) + set(GEN_DOC_FILE ${ONNX_MLIR_BIN_ROOT}/docs/Dialects/${dialect}.md) add_custom_command( OUTPUT ${GEN_DOC_FILE} COMMAND ${CMAKE_COMMAND} -E copy diff --git a/doc/ImportONNXDefs.md b/doc/ImportONNXDefs.md deleted file mode 100644 index 0172919..0000000 --- a/doc/ImportONNXDefs.md +++ /dev/null @@ -1,32 +0,0 @@ -# Import ONNX specifications into ONNX MLIR -The specifications of ONNX are defined under onnx/defs directory in ONNX projects. -There is a python script onnx/defs/gen_doc.py that automatically generate documents about operations in ONNX (docs/Operations.md). -ONNX MLIR modified this script to import ONNX specifications into ONNX MLIR. There are two files generated for ONNX MLIR with the modified gen_doc.py: -1. src/Dialect/ONNX/ONNXOps.td.inc: Operation defintion for MLIR tablegen. Will be included in src/Dialect/ONNX/ONNXOps.td -2. src/Builder/OpBuildTable.inc: c code for ONNX MLIR frontend to import operation nodes from ONNX model. Will be included in src/Builder/FrontendDialectTransformer.cpp - -## How to use the script -1. Get ONNX. You can use onnx-mlir/third_party/onnx -2. Perform the following steps (assume that we use onnx-mlir/third_party/onnx): -```bash -$ cd onnx-mlir -$ cp doc/gen_doc.py third_party/onnx/onnx/defs/ -$ cd third_party/onnx -$ python onnx/defs/gen_doc.py -$ cd ../.. -$ cp third_party/onnx/onnx/defs/ONNXOps.td.inc src/Dialect/ONNX/ -$ cp third_party/onnx/onnx/defs/OpBuildTable.inc src/Builder/ -``` - -## Consistency -The Operators.md generated by gen_doc.py is copied into doc. Please refer to this specification, not the one in onnx github, to make sure operators are consistent in version with ONNXOps.td.inc. - -## Customization -In addition to following the ONNX specification, the modified gen_doc.py provides some mechanism for you to customize the output. -Several tables are defined at the beginning of the script: -1. special_attr_defaults: gives attribute special default value. -2. special_op_handler: creates special import function in frontend_dialect_transformer.cpp. Currently special handler is used for operations with oprational arguments -3. OpsWithShapeInference: list of operations which have shape inference defined -4. OpsWithCanonicalizer: list of operations which have canonical form -5. OpsWithPromotableConstOperands: list of operations which have operands that, if produced by constant operations, should be promoted to become an attribute (via attribute promotion) -6. custom_builder_ops_list: list of operations which need custom build methods to deduce result types diff --git a/doc/CMakeLists.txt b/docs/CMakeLists.txt similarity index 100% rename from doc/CMakeLists.txt rename to docs/CMakeLists.txt diff --git a/doc/Dialects/onnx.md b/docs/Dialects/onnx.md similarity index 100% rename from doc/Dialects/onnx.md rename to docs/Dialects/onnx.md diff --git a/docs/Documentation.md b/docs/Documentation.md new file mode 100644 index 0000000..3748cf8 --- /dev/null +++ b/docs/Documentation.md @@ -0,0 +1,14 @@ +# About Documentation + +## How to add a new documentation page + +Firstly, `/docs` is the root directory of the documentation website, meaning that any +documentation page you wish to display to a user must be located within `/docs`. + +Secondly, add the documentation page into the navigation configuration file located at +`/docs/_data/navigation.yaml`. Edit the table of content to include the path to +the newly created documentation page with a descriptive title. + +Then, capture the changes done in a patch and submit a pull request; once the patch is +merged into `onnx-mlir` codebase, a link pointing to the file path specified with the +descriptive title you provided will appear on the navigation panel. \ No newline at end of file diff --git a/docs/ImportONNXDefs.md b/docs/ImportONNXDefs.md new file mode 100644 index 0000000..e78abd4 --- /dev/null +++ b/docs/ImportONNXDefs.md @@ -0,0 +1,32 @@ +# Import ONNX specifications into ONNX-MLIR + +ONNX specifications are defined under `onnx/defs` directory in the ONNX project repository. +There is a python script onnx/defs/gen_doc.py that automatically generate documents about operations in ONNX (docs/Operations.md). +ONNX-MLIR modified this script to import ONNX specifications into ONNX-MLIR. +There are two files generated for ONNX MLIR with the modified gen_doc.py: + +1. `src/Dialect/ONNX/ONNXOps.td.inc`: Operation definition for MLIR TableGen. `src/Dialect/ONNX/ONNXOps.td` includes this file. +2. `src/Builder/OpBuildTable.inc`: C++ code for ONNX-MLIR frontend to import operation nodes from ONNX model. `src/Builder/FrontendDialectTransformer.cpp` includes this file. + +## How to use the script +1. Install [ONNX](https://github.com/onnx/onnx). We highly recommend that you use the one located at `third_party/onnx.` +2. Make target `OMONNXOpsIncTranslation`. For example, +``` +make OMONNXOpsIncTranslation +```` +Target `OMONNXOpsIncTranslation` invokes the script and places the generated files into the correct directories correspondingly. + +## Consistency +For reference to the schema and semantics of an operation, please refer to [ONNX Dialect](Dialects/onnx.md). +Even though we strive to support the latest version of ONNX specification as quickly as we can, there will inevitably be a delay between the introduction of new changes in the ONNX specification and the adoption in our codebase. +Due to the possibility of such a delay, operator definition within the ONNX project repository may describe features and schemas that we do not yet support. + +## Customization +In addition to following the ONNX specification, the modified gen_doc.py provides some mechanism for you to customize the output. +Several tables are defined at the beginning of the script: +1. `special_attr_defaults`: gives attribute special default value. +2. `special_op_handler`: creates special import function in frontend_dialect_transformer.cpp. Currently, a special handler is used for operations with operational arguments +3. `OpsWithShapeInference`: list of operations which have shape inference defined +4. `OpsWithCanonicalizer`: list of operations which have a canonical form +5. `OpsWithPromotableConstOperands`: list of operations which have operands that, if produced by constant operations, should be promoted to become an attribute (via attribute promotion) +6. `custom_builder_ops_list`: list of operations which need custom build methods to deduce result types diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..bc4f421 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,106 @@ +# ONNX MLIR +The Open Neural Network Exchange implementation in MLIR. + +[![CircleCI](https://circleci.com/gh/onnx/onnx-mlir/tree/master.svg?style=svg)](https://circleci.com/gh/onnx/onnx-mlir/tree/master) + +## Prerequisites + +``` +gcc >= 6.4 +libprotoc >= 3.11.0 +cmake >= 3.15.4 +``` + +## Installation + +Firstly, install MLIR (as a part of LLVM-Project): + +[same-as-file]: <> (utils/install-mlir.sh) +``` bash +git clone https://github.com/llvm/llvm-project.git +# Check out a specific branch that is known to work with ONNX MLIR. +cd llvm-project && git checkout 07e462526d0cbae40b320e1a4307ce11e197fb0a && cd .. +mkdir llvm-project/build +cd llvm-project/build +cmake -G Ninja ../llvm \ + -DLLVM_ENABLE_PROJECTS=mlir \ + -DLLVM_BUILD_EXAMPLES=ON \ + -DLLVM_TARGETS_TO_BUILD="host" \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_ENABLE_ASSERTIONS=ON \ + -DLLVM_ENABLE_RTTI=ON + +cmake --build . --target -- ${MAKEFLAGS} +cmake --build . --target check-mlir +``` + +Two environment variables need to be set: +- LLVM_PROJ_SRC should point to the llvm-project src directory (e.g., llvm-project/). +- LLVM_PROJ_BUILD should point to the llvm-project build directory (e.g., llvm-project/build). + +To build ONNX-MLIR, use the following command: + +[same-as-file]: <> ({"ref": "utils/install-onnx-mlir.sh", "skip-doc": 2}) +``` +git clone --recursive https://github.com/onnx/onnx-mlir.git + +# Export environment variables pointing to LLVM-Projects. +export LLVM_PROJ_SRC=$(pwd)/llvm-project/ +export LLVM_PROJ_BUILD=$(pwd)/llvm-project/build + +mkdir onnx-mlir/build && cd onnx-mlir/build +cmake .. +cmake --build . --target onnx-mlir + +# Run FileCheck tests: +export LIT_OPTS=-v +cmake --build . --target check-onnx-lit +``` + +After the above commands succeed, an `onnx-mlir` executable should appear in the `bin` directory. + +## Using ONNX MLIR + +The usage of `onnx-mlir` is as such: +``` +OVERVIEW: ONNX MLIR modular optimizer driver + +USAGE: onnx-mlir [options] + +OPTIONS: + +Generic Options: + + --help - Display available options (--help-hidden for more) + --help-list - Display list of available options (--help-list-hidden for more) + --version - Display the version of this program + +ONNX MLIR Options: +These are frontend options. + + Choose target to emit: + --EmitONNXIR - Ingest ONNX and emit corresponding ONNX dialect. + --EmitMLIR - Lower model to MLIR built-in transformation dialect. + --EmitLLVMIR - Lower model to LLVM IR (LLVM dialect). + --EmitLLVMBC - Lower model to LLVM IR and emit (to file) LLVM bitcode for model. +``` + +## Example + +For example, to lower an ONNX model (e.g., add.onnx) to ONNX dialect, use the following command: +``` +./onnx-mlir --EmitONNXIR add.onnx +``` +The output should look like: +``` +module { + func @main_graph(%arg0: tensor<10x10x10xf32>, %arg1: tensor<10x10x10xf32>) -> tensor<10x10x10xf32> { + %0 = "onnx.Add"(%arg0, %arg1) : (tensor<10x10x10xf32>, tensor<10x10x10xf32>) -> tensor<10x10x10xf32> + return %0 : tensor<10x10x10xf32> + } +} +``` + +## Troubleshooting + +If the latest LLVM project fails to work due to the latest changes to the MLIR subproject please consider using a slightly older version of LLVM. One such version, which we use, can be found [here](https://github.com/clang-ykt/llvm-project). diff --git a/docs/_config.yml b/docs/_config.yml new file mode 100644 index 0000000..322d2a9 --- /dev/null +++ b/docs/_config.yml @@ -0,0 +1,7 @@ +theme: jekyll-theme-minimal +defaults: + - + scope: + path: "README.md" + values: + permalink: "README.html" \ No newline at end of file diff --git a/docs/_data/navigation.yml b/docs/_data/navigation.yml new file mode 100644 index 0000000..07986a6 --- /dev/null +++ b/docs/_data/navigation.yml @@ -0,0 +1,33 @@ +toc: +# Restore when we have a proper +# overview & installation page. +# - title: First Steps +# subfolderitems: +# - page: Overview +# url: /thing1.html +# - page: Installation +# url: /thing2.html +# - title: Tutorials +# subfolderitems: +# - page: Placeholder +# url: /piece1.html +# - title: How-tos +# subfolderitems: +# - page: Placeholder +# url: /piece1.html + - title: References + subfolderitems: + - page: ONNX Dialect + url: /Dialects/onnx.html + - page: Generating ONNX Dialect + url: /ImportONNXDefs.html + - page: About Documentation + url: /Documentation.html +# - title: Discussions +# subfolderitems: +# - page: Placeholder +# url: /piece1.html + - title: Tools + subfolderitems: + - page: DocCheck - Handling Necessary Code Duplication + url: /doc_check/ \ No newline at end of file diff --git a/docs/_layouts/default.html b/docs/_layouts/default.html new file mode 100644 index 0000000..8514963 --- /dev/null +++ b/docs/_layouts/default.html @@ -0,0 +1,74 @@ + + + + + + + +{% seo %} + + + + +
+
+

{{ site.title | default: site.github.repository_name }}

+ + {% if site.logo %} + Logo + {% endif %} + +

{{ site.description | default: site.github.project_tagline }}

+ + {% if site.github.is_project_page %} +

View the Project on GitHub {{ site.github.repository_nwo }}

+ {% endif %} + + {% if site.github.is_user_page %} +

View My GitHub Profile

+ {% endif %} + + {% if site.show_downloads %} + + {% endif %} + + {% for item in site.data.navigation.toc %} +

{{ item.title }}

+ {% for entry in item.subfolderitems %} +
{{ entry.page }}
+ {% endfor %} + {% endfor %} +
+ {% if site.github.is_project_page %} +

This project is maintained by {{ site.github.owner_name }}

+ {% endif %} +

Hosted on GitHub Pages — Theme by orderedlist

+
+
+ + {{ content }} + +
+ +
+ + {% if site.google_analytics %} + + {% endif %} + + diff --git a/doc/doc_check/CMakeLists.txt b/docs/doc_check/CMakeLists.txt similarity index 74% rename from doc/doc_check/CMakeLists.txt rename to docs/doc_check/CMakeLists.txt index 988bfa4..95eec9d 100644 --- a/doc/doc_check/CMakeLists.txt +++ b/docs/doc_check/CMakeLists.txt @@ -4,5 +4,5 @@ add_custom_target(check-doc COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/check.py ${ONNX_MLIR_SRC_ROOT} - --exclude_dirs third_party doc/doc_check/test) + --exclude_dirs third_party docs/doc_check/test) diff --git a/doc/doc_check/README.md b/docs/doc_check/README.md similarity index 94% rename from doc/doc_check/README.md rename to docs/doc_check/README.md index 61923ff..28b7acc 100644 --- a/doc/doc_check/README.md +++ b/docs/doc_check/README.md @@ -58,7 +58,7 @@ to ensure the code snippet is always the same as its copy used in some unit test be fully specified using the name of the reference file to check against. For example, to ensure a code snippet is the same as a unit-tested file `reference.cpp`, use the following directive as shown in the documentation snippet: -[same-as-file]: <> (doc/doc_check/test/same-as-file/simple/README.md) +[same-as-file]: <> (docs/doc_check/test/same-as-file/simple/README.md) ````markdown Lorem ipsum dolor sit amet, consectetur adipiscing elit. @@ -85,7 +85,7 @@ In the canonical form of directive configuration (as a python dictionary literal For example, to ensure the following code snippet is the same as a unit-tested file `reference.cpp`, except for the first 2 lines of the code used in documentation, and the first 3 lines of code used in the reference file, the following directive configuration can be used: -[same-as-file]: <> (doc/doc_check/test/same-as-file/skip-doc-ref/README.md) +[same-as-file]: <> (docs/doc_check/test/same-as-file/skip-doc-ref/README.md) ````markdown Lorem ipsum dolor sit amet, consectetur adipiscing elit. @@ -115,13 +115,13 @@ This directive supports these parameters in it: For example, to ensure that the content of a file `test.in`: -[same-as-file]: <> (doc/doc_check/test/file-same-as-stdout/success/test.in) +[same-as-file]: <> (docs/doc_check/test/file-same-as-stdout/success/test.in) ``` dog ``` is exactly the same as the output of command execution `echo dog`, one can use the following directive: -[same-as-file]: <> (doc/doc_check/test/file-same-as-stdout/success/test.in.dc) +[same-as-file]: <> (docs/doc_check/test/file-same-as-stdout/success/test.in.dc) ``` file-same-as-stdout({"file": "test.in", "cmd": ["echo", "dog"]}) ``` \ No newline at end of file diff --git a/doc/doc_check/check.py b/docs/doc_check/check.py similarity index 100% rename from doc/doc_check/check.py rename to docs/doc_check/check.py diff --git a/doc/doc_check/directive.py b/docs/doc_check/directive.py similarity index 100% rename from doc/doc_check/directive.py rename to docs/doc_check/directive.py diff --git a/doc/doc_check/directive_impl/file_same_as_stdout.py b/docs/doc_check/directive_impl/file_same_as_stdout.py similarity index 100% rename from doc/doc_check/directive_impl/file_same_as_stdout.py rename to docs/doc_check/directive_impl/file_same_as_stdout.py diff --git a/doc/doc_check/directive_impl/same_as_file.py b/docs/doc_check/directive_impl/same_as_file.py similarity index 100% rename from doc/doc_check/directive_impl/same_as_file.py rename to docs/doc_check/directive_impl/same_as_file.py diff --git a/doc/doc_check/doc_parser.py b/docs/doc_check/doc_parser.py similarity index 100% rename from doc/doc_check/doc_parser.py rename to docs/doc_check/doc_parser.py diff --git a/doc/doc_check/test/file-same-as-stdout/failure/test.in b/docs/doc_check/test/file-same-as-stdout/failure/test.in similarity index 100% rename from doc/doc_check/test/file-same-as-stdout/failure/test.in rename to docs/doc_check/test/file-same-as-stdout/failure/test.in diff --git a/doc/doc_check/test/file-same-as-stdout/failure/test.in.dc b/docs/doc_check/test/file-same-as-stdout/failure/test.in.dc similarity index 100% rename from doc/doc_check/test/file-same-as-stdout/failure/test.in.dc rename to docs/doc_check/test/file-same-as-stdout/failure/test.in.dc diff --git a/doc/doc_check/test/file-same-as-stdout/success/test.in b/docs/doc_check/test/file-same-as-stdout/success/test.in similarity index 100% rename from doc/doc_check/test/file-same-as-stdout/success/test.in rename to docs/doc_check/test/file-same-as-stdout/success/test.in diff --git a/doc/doc_check/test/file-same-as-stdout/success/test.in.dc b/docs/doc_check/test/file-same-as-stdout/success/test.in.dc similarity index 100% rename from doc/doc_check/test/file-same-as-stdout/success/test.in.dc rename to docs/doc_check/test/file-same-as-stdout/success/test.in.dc diff --git a/doc/doc_check/test/same-as-file/error-doc-different-from-ref/README.md b/docs/doc_check/test/same-as-file/error-doc-different-from-ref/README.md similarity index 100% rename from doc/doc_check/test/same-as-file/error-doc-different-from-ref/README.md rename to docs/doc_check/test/same-as-file/error-doc-different-from-ref/README.md diff --git a/doc/doc_check/test/same-as-file/error-doc-different-from-ref/reference.cpp b/docs/doc_check/test/same-as-file/error-doc-different-from-ref/reference.cpp similarity index 100% rename from doc/doc_check/test/same-as-file/error-doc-different-from-ref/reference.cpp rename to docs/doc_check/test/same-as-file/error-doc-different-from-ref/reference.cpp diff --git a/doc/doc_check/test/same-as-file/error-doc-shorter-than-ref/README.md b/docs/doc_check/test/same-as-file/error-doc-shorter-than-ref/README.md similarity index 100% rename from doc/doc_check/test/same-as-file/error-doc-shorter-than-ref/README.md rename to docs/doc_check/test/same-as-file/error-doc-shorter-than-ref/README.md diff --git a/doc/doc_check/test/same-as-file/error-doc-shorter-than-ref/reference.cpp b/docs/doc_check/test/same-as-file/error-doc-shorter-than-ref/reference.cpp similarity index 100% rename from doc/doc_check/test/same-as-file/error-doc-shorter-than-ref/reference.cpp rename to docs/doc_check/test/same-as-file/error-doc-shorter-than-ref/reference.cpp diff --git a/doc/doc_check/test/same-as-file/simple/README.md b/docs/doc_check/test/same-as-file/simple/README.md similarity index 100% rename from doc/doc_check/test/same-as-file/simple/README.md rename to docs/doc_check/test/same-as-file/simple/README.md diff --git a/doc/doc_check/test/same-as-file/simple/reference.cpp b/docs/doc_check/test/same-as-file/simple/reference.cpp similarity index 100% rename from doc/doc_check/test/same-as-file/simple/reference.cpp rename to docs/doc_check/test/same-as-file/simple/reference.cpp diff --git a/doc/doc_check/test/same-as-file/skip-doc-ref/README.md b/docs/doc_check/test/same-as-file/skip-doc-ref/README.md similarity index 100% rename from doc/doc_check/test/same-as-file/skip-doc-ref/README.md rename to docs/doc_check/test/same-as-file/skip-doc-ref/README.md diff --git a/doc/doc_check/test/same-as-file/skip-doc-ref/reference.cpp b/docs/doc_check/test/same-as-file/skip-doc-ref/reference.cpp similarity index 100% rename from doc/doc_check/test/same-as-file/skip-doc-ref/reference.cpp rename to docs/doc_check/test/same-as-file/skip-doc-ref/reference.cpp diff --git a/doc/doc_check/test/test_file-same-as-stdout.py b/docs/doc_check/test/test_file-same-as-stdout.py similarity index 100% rename from doc/doc_check/test/test_file-same-as-stdout.py rename to docs/doc_check/test/test_file-same-as-stdout.py diff --git a/doc/doc_check/test/test_same-as-file.py b/docs/doc_check/test/test_same-as-file.py similarity index 100% rename from doc/doc_check/test/test_same-as-file.py rename to docs/doc_check/test/test_same-as-file.py diff --git a/doc/doc_check/utils.py b/docs/doc_check/utils.py similarity index 100% rename from doc/doc_check/utils.py rename to docs/doc_check/utils.py diff --git a/src/Builder/OpBuildTable.inc b/src/Builder/OpBuildTable.inc index e876398..c37b59b 100644 --- a/src/Builder/OpBuildTable.inc +++ b/src/Builder/OpBuildTable.inc @@ -1,7 +1,7 @@ //******************************************************** // Do not modify this file directly. // This file is automatically generated via script. -// Details can be found in doc/readonnxdefs.md . +// Details can be found in docs/readonnxdefs.md . //******************************************************** if (opName == "Abs") diff --git a/src/Dialect/ONNX/ONNXOps.td.inc b/src/Dialect/ONNX/ONNXOps.td.inc index 8476986..2a7c34a 100644 --- a/src/Dialect/ONNX/ONNXOps.td.inc +++ b/src/Dialect/ONNX/ONNXOps.td.inc @@ -1,7 +1,7 @@ //******************************************************** // Do not modify this file directly. // This file is automatically generated via script. -// Details can be found in doc/readonnxdefs.md . +// Details can be found in docs/readonnxdefs.md . //******************************************************** def ONNXAbsOp:ONNX_Op<"Abs", diff --git a/utils/gen_doc.py b/utils/gen_doc.py index db52898..075dd36 100644 --- a/utils/gen_doc.py +++ b/utils/gen_doc.py @@ -533,7 +533,7 @@ def main(args): # type: (Type[Args]) -> None '//********************************************************\n' '// Do not modify this file directly.\n' '// This file is automatically generated via script.\n' - '// Details can be found in doc/readonnxdefs.md .\n' + '// Details can be found in docs/readonnxdefs.md .\n' '//********************************************************\n\n') autogen_warning = autogen_warning.format(curr_utc_time)