save and check the version of ONNX operation (#125)
* add version control * fix the comments * response to review and add onnx_ml * update document * add target and rename gen_doc.py * small fix
This commit is contained in:
parent
ad61eee908
commit
a98c723b11
|
@ -22,7 +22,7 @@ Even though we strive to support the latest version of ONNX specification as qui
|
|||
Due to the possibility of such a delay, operator definition within the ONNX project repository may describe features and schemas that we do not yet support.
|
||||
|
||||
## Customization
|
||||
In addition to following the ONNX specification, the modified gen_doc.py provides some mechanism for you to customize the output.
|
||||
In addition to following the ONNX specification, the script gen_onnx_mlir.py, modified gen_doc.py, provides some mechanism for you to customize the output.
|
||||
Several tables are defined at the beginning of the script:
|
||||
1. `special_attr_defaults`: gives attribute special default value.
|
||||
2. `special_op_handler`: creates special import function in frontend_dialect_transformer.cpp. Currently, a special handler is used for operations with operational arguments
|
||||
|
@ -30,3 +30,8 @@ Several tables are defined at the beginning of the script:
|
|||
4. `OpsWithCanonicalizer`: list of operations which have a canonical form
|
||||
5. `OpsWithPromotableConstOperands`: list of operations which have operands that, if produced by constant operations, should be promoted to become an attribute (via attribute promotion)
|
||||
6. `custom_builder_ops_list`: list of operations which need custom build methods to deduce result types
|
||||
|
||||
## Version of Operations
|
||||
As stated previous, we try to support the latest version of ONNX operations. The version of each operation currently supported is recorded in gen_onnx_mlir.py. This mechanism provides some stability in version. To check the changes in version, run gen_onnx_mlir.py with flag "--check-version" and the changes will be reported. To move to a newer version, manually update the version dictionary in the script.
|
||||
Supporting mulitple versions of one operation is not available yet.
|
||||
|
||||
|
|
|
@ -1 +1 @@
|
|||
file-same-as-stdout({"file": "src/Builder/OpBuildTable.inc", "cmd": ["python", "utils/gen_doc.py", "--dry-run-op-build-table"]})
|
||||
file-same-as-stdout({"file": "src/Builder/OpBuildTable.inc", "cmd": ["python", "utils/gen_onnx_mlir.py", "--dry-run-op-build-table"]})
|
||||
|
|
|
@ -1 +1 @@
|
|||
file-same-as-stdout({"file": "src/Dialect/ONNX/ONNXOps.td.inc", "cmd": ["python", "utils/gen_doc.py", "--dry-run-onnx-ops"]})
|
||||
file-same-as-stdout({"file": "src/Dialect/ONNX/ONNXOps.td.inc", "cmd": ["python", "utils/gen_onnx_mlir.py", "--dry-run-onnx-ops"]})
|
||||
|
|
|
@ -1,8 +1,8 @@
|
|||
# Invoke gen_doc.py to obtain ONNXOps.td.inc, OpBuildTable.inc.
|
||||
# Invoke gen_onnx_mlir.py to obtain ONNXOps.td.inc, OpBuildTable.inc.
|
||||
add_custom_command(OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/ONNXOps.td.inc
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/OpBuildTable.inc
|
||||
COMMAND python ${CMAKE_CURRENT_SOURCE_DIR}/gen_doc.py
|
||||
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/gen_doc.py)
|
||||
COMMAND python ${CMAKE_CURRENT_SOURCE_DIR}/gen_onnx_mlir.py
|
||||
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/gen_onnx_mlir.py)
|
||||
|
||||
# Move the generated files to respective destinations:
|
||||
# ONNXOps.td.inc -> src/Dialect/ONNX/ONNXOps.td.inc
|
||||
|
@ -23,11 +23,11 @@ add_custom_target(OMONNXOpsIncTranslation
|
|||
DEPENDS OMONNXOpsTableGenIncGen
|
||||
OMONNXOpsBuildTableIncGen)
|
||||
|
||||
# Invoke gen_doc.py to obtain ONNXOps.td.inc, OpBuildTable.inc.
|
||||
# Invoke gen_onnx_mlir.py to obtain ONNXOps.td.inc, OpBuildTable.inc.
|
||||
add_custom_command(OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/MLONNXOps.td.inc
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/MLOpBuildTable.inc
|
||||
COMMAND python ${CMAKE_CURRENT_SOURCE_DIR}/gen_doc.py --domain="ONNX_ML"
|
||||
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/gen_doc.py)
|
||||
COMMAND python ${CMAKE_CURRENT_SOURCE_DIR}/gen_onnx_mlir.py --domain="ONNX_ML"
|
||||
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/gen_onnx_mlir.py)
|
||||
|
||||
# Move the generated files to respective destinations:
|
||||
# MLONNXOps.td.inc -> src/Dialect/MLONNX/MLONNXOps.td.inc
|
||||
|
@ -47,3 +47,11 @@ add_custom_target(OMMLONNXOpsBuildTableIncGen
|
|||
add_custom_target(OMMLONNXOpsIncTranslation
|
||||
DEPENDS OMMLONNXOpsTableGenIncGen
|
||||
OMMLONNXOpsBuildTableIncGen)
|
||||
|
||||
add_custom_target(OMONNXCheckVersion
|
||||
COMMAND python ${CMAKE_CURRENT_SOURCE_DIR}/gen_onnx_mlir.py --check-operation-version)
|
||||
|
||||
add_custom_target(OMMLONNXCheckVersion
|
||||
COMMAND python ${CMAKE_CURRENT_SOURCE_DIR}/gen_onnx_mlir.py
|
||||
--check-operation-version --domain="ONNX_ML")
|
||||
|
||||
|
|
|
@ -20,6 +20,8 @@ from onnx.backend.test.case import collect_snippets
|
|||
from onnx.backend.sample.ops import collect_sample_implementations
|
||||
from typing import Any, Text, Sequence, Dict, List, Type, Set, Tuple
|
||||
|
||||
import pprint
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--dry-run-onnx-ops",
|
||||
help="Output ONNXOps.td.inc content to stdout.",
|
||||
|
@ -29,12 +31,201 @@ parser.add_argument("--dry-run-op-build-table",
|
|||
help="Output OpBuildTable.inc content to stdout.",
|
||||
action="store_true",
|
||||
default=False)
|
||||
parser.add_argument("--check-operation-version",
|
||||
help="check whether the imported onnx package has new operation or "
|
||||
" newer version of operation compared with version stored in version_dicts",
|
||||
action="store_true",
|
||||
default=False)
|
||||
parser.add_argument("--domain",
|
||||
help="specify domain, ONNX or ONNX_ML",
|
||||
default = "ONNX")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
check_operation_version = args.check_operation_version
|
||||
|
||||
|
||||
# Record the version of each operation that is treated as the current version.
|
||||
# To check whether the onnx package being used has newer version operation,
|
||||
# run this script with --check-operation-version flag.
|
||||
# Update this dictionary when a newer version is implemented
|
||||
# TODO: how to keep the old version
|
||||
onnx_version_dict = {'Abs': 6,
|
||||
'Acos': 7,
|
||||
'Acosh': 9,
|
||||
'Add': 7,
|
||||
'And': 7,
|
||||
'ArgMax': 11,
|
||||
'ArgMin': 11,
|
||||
'Asin': 7,
|
||||
'Asinh': 9,
|
||||
'Atan': 7,
|
||||
'Atanh': 9,
|
||||
'AveragePool': 11,
|
||||
'BatchNormalization': 9,
|
||||
'BitShift': 11,
|
||||
'Cast': 9,
|
||||
'Ceil': 6,
|
||||
'Clip': 11,
|
||||
'Compress': 11,
|
||||
'Concat': 11,
|
||||
'ConcatFromSequence': 11,
|
||||
'Constant': 11,
|
||||
'ConstantOfShape': 9,
|
||||
'Conv': 11,
|
||||
'ConvInteger': 10,
|
||||
'ConvTranspose': 11,
|
||||
'Cos': 7,
|
||||
'Cosh': 9,
|
||||
'CumSum': 11,
|
||||
'DepthToSpace': 11,
|
||||
'DequantizeLinear': 10,
|
||||
'Det': 11,
|
||||
'Div': 7,
|
||||
'Dropout': 10,
|
||||
'DynamicQuantizeLinear': 11,
|
||||
'Elu': 6,
|
||||
'Equal': 11,
|
||||
'Erf': 9,
|
||||
'Exp': 6,
|
||||
'Expand': 8,
|
||||
'EyeLike': 9,
|
||||
'Flatten': 11,
|
||||
'Floor': 6,
|
||||
'GRU': 7,
|
||||
'Gather': 11,
|
||||
'GatherElements': 11,
|
||||
'GatherND': 11,
|
||||
'Gemm': 11,
|
||||
'GlobalAveragePool': 1,
|
||||
'GlobalLpPool': 2,
|
||||
'GlobalMaxPool': 1,
|
||||
'Greater': 9,
|
||||
'HardSigmoid': 6,
|
||||
'Hardmax': 11,
|
||||
'Identity': 1,
|
||||
'If': 11,
|
||||
'InstanceNormalization': 6,
|
||||
'IsInf': 10,
|
||||
'IsNaN': 9,
|
||||
'LRN': 1,
|
||||
'LSTM': 7,
|
||||
'LeakyRelu': 6,
|
||||
'Less': 9,
|
||||
'Log': 6,
|
||||
'LogSoftmax': 11,
|
||||
'Loop': 11,
|
||||
'LpNormalization': 1,
|
||||
'LpPool': 11,
|
||||
'MatMul': 9,
|
||||
'MatMulInteger': 10,
|
||||
'Max': 8,
|
||||
'MaxPool': 11,
|
||||
'MaxRoiPool': 1,
|
||||
'MaxUnpool': 11,
|
||||
'Mean': 8,
|
||||
'MeanVarianceNormalization': 9,
|
||||
'Min': 8,
|
||||
'Mod': 10,
|
||||
'Mul': 7,
|
||||
'Multinomial': 7,
|
||||
'Neg': 6,
|
||||
'NonMaxSuppression': 11,
|
||||
'NonZero': 9,
|
||||
'Not': 1,
|
||||
'OneHot': 11,
|
||||
'Or': 7,
|
||||
'PRelu': 9,
|
||||
'Pad': 11,
|
||||
'Pow': 7,
|
||||
'QLinearConv': 10,
|
||||
'QLinearMatMul': 10,
|
||||
'QuantizeLinear': 10,
|
||||
'RNN': 7,
|
||||
'RandomNormal': 1,
|
||||
'RandomNormalLike': 1,
|
||||
'RandomUniform': 1,
|
||||
'RandomUniformLike': 1,
|
||||
'Range': 11,
|
||||
'Reciprocal': 6,
|
||||
'ReduceL1': 11,
|
||||
'ReduceL2': 11,
|
||||
'ReduceLogSum': 11,
|
||||
'ReduceLogSumExp': 11,
|
||||
'ReduceMax': 11,
|
||||
'ReduceMean': 11,
|
||||
'ReduceMin': 11,
|
||||
'ReduceProd': 11,
|
||||
'ReduceSum': 11,
|
||||
'ReduceSumSquare': 11,
|
||||
'Relu': 6,
|
||||
'Reshape': 5,
|
||||
'Resize': 11,
|
||||
'ReverseSequence': 10,
|
||||
'RoiAlign': 10,
|
||||
'Round': 11,
|
||||
'Scan': 11,
|
||||
'Scatter': 11,
|
||||
'ScatterElements': 11,
|
||||
'ScatterND': 11,
|
||||
'Selu': 6,
|
||||
'SequenceAt': 11,
|
||||
'SequenceConstruct': 11,
|
||||
'SequenceEmpty': 11,
|
||||
'SequenceErase': 11,
|
||||
'SequenceInsert': 11,
|
||||
'SequenceLength': 11,
|
||||
'Shape': 1,
|
||||
'Shrink': 9,
|
||||
'Sigmoid': 6,
|
||||
'Sign': 9,
|
||||
'Sin': 7,
|
||||
'Sinh': 9,
|
||||
'Size': 1,
|
||||
'Slice': 11,
|
||||
'Softmax': 11,
|
||||
'Softplus': 1,
|
||||
'Softsign': 1,
|
||||
'SpaceToDepth': 1,
|
||||
'Split': 11,
|
||||
'SplitToSequence': 11,
|
||||
'Sqrt': 6,
|
||||
'Squeeze': 11,
|
||||
'StringNormalizer': 10,
|
||||
'Sub': 7,
|
||||
'Sum': 8,
|
||||
'Tan': 7,
|
||||
'Tanh': 6,
|
||||
'TfIdfVectorizer': 9,
|
||||
'ThresholdedRelu': 10,
|
||||
'Tile': 6,
|
||||
'TopK': 11,
|
||||
'Transpose': 1,
|
||||
'Unique': 11,
|
||||
'Unsqueeze': 11,
|
||||
'Upsample': 10,
|
||||
'Where': 9,
|
||||
'Xor': 7}
|
||||
|
||||
onnx_ml_version_dict = {'ArrayFeatureExtractor': 1,
|
||||
'Binarizer': 1,
|
||||
'CastMap': 1,
|
||||
'CategoryMapper': 1,
|
||||
'DictVectorizer': 1,
|
||||
'FeatureVectorizer': 1,
|
||||
'Imputer': 1,
|
||||
'LabelEncoder': 2,
|
||||
'LinearClassifier': 1,
|
||||
'LinearRegressor': 1,
|
||||
'Normalizer': 1,
|
||||
'OneHotEncoder': 1,
|
||||
'SVMClassifier': 1,
|
||||
'SVMRegressor': 1,
|
||||
'Scaler': 1,
|
||||
'TreeEnsembleClassifier': 1,
|
||||
'TreeEnsembleRegressor': 1,
|
||||
'ZipMap': 1}
|
||||
|
||||
# Manual specification of attribute defaults.
|
||||
special_attr_defaults = dict([
|
||||
# ("AveragePool.kernel_shape", ('ints', '{}')),
|
||||
|
@ -536,7 +727,10 @@ def build_operator_schemas():
|
|||
for domain, _supportmap in sorted(index.items()):
|
||||
if not should_render_domain(domain):
|
||||
continue
|
||||
|
||||
if domain == ONNX_ML_DOMAIN:
|
||||
version_dict = onnx_ml_version_dict
|
||||
else:
|
||||
version_dict = onnx_version_dict
|
||||
processed_supportmap = list()
|
||||
for _support, _namemap in sorted(_supportmap.items()):
|
||||
processed_namemap = list()
|
||||
|
@ -546,8 +740,36 @@ def build_operator_schemas():
|
|||
schema = versions[-1]
|
||||
if schema.name in exsting_ops:
|
||||
continue
|
||||
exsting_ops.add(schema.name)
|
||||
processed_namemap.append((n, schema, versions))
|
||||
|
||||
if check_operation_version :
|
||||
# Generate operation of the latest version of your onnx.
|
||||
exsting_ops.add(schema.name)
|
||||
processed_namemap.append((n, schema, versions))
|
||||
|
||||
# Add checks against version_dict
|
||||
if schema.name not in version_dict :
|
||||
print("Check-operation-version: Operation {} with version is new".format(
|
||||
schema.since_version, schema.name))
|
||||
elif schema.since_version > version_dict[schema.name]:
|
||||
print("Check-operation-version: Operation {} has a newer version {}"+
|
||||
"(old version {})".format( schema.name,
|
||||
schema.since_version, version_dict[schema.name]))
|
||||
else:
|
||||
# Generate operation according to the version in version_dict.
|
||||
if schema.name not in version_dict :
|
||||
continue
|
||||
found = False
|
||||
for schema in reversed(versions):
|
||||
# Check the version number against the version_dict
|
||||
if schema.since_version == version_dict[schema.name]:
|
||||
exsting_ops.add(schema.name)
|
||||
processed_namemap.append((n, schema, versions))
|
||||
found = True
|
||||
break
|
||||
if not found:
|
||||
print("Your onnx may be too old."
|
||||
"right version for opertion {} not found".format(
|
||||
schema.name))
|
||||
processed_supportmap.append((_support, processed_namemap))
|
||||
operator_schemas.append((domain, processed_supportmap))
|
||||
return operator_schemas
|
||||
|
@ -570,12 +792,18 @@ def main(args): # type: (Type[Args]) -> None
|
|||
op_importer = args.op_importer
|
||||
op_importer.write(autogen_warning)
|
||||
|
||||
version_dict = dict()
|
||||
for domain, supportmap in build_operator_schemas():
|
||||
for _, namemap in supportmap:
|
||||
for op_type, schema, versions in namemap:
|
||||
gen_op_importer(schema, op_importer)
|
||||
r = gen_op_def(schema)
|
||||
op_def.write(r)
|
||||
if check_operation_version:
|
||||
version_dict[schema.name] = schema.since_version
|
||||
else:
|
||||
gen_op_importer(schema, op_importer)
|
||||
r = gen_op_def(schema)
|
||||
op_def.write(r)
|
||||
if check_operation_version :
|
||||
pprint.pprint(version_dict)
|
||||
|
||||
if __name__ == '__main__':
|
||||
curr_dir = os.path.dirname(os.path.realpath(__file__))
|
Loading…
Reference in New Issue