Merge pull request #996 from xiebaiyuan/develop

add some python tools for paddlemobile

Author: xiebaiyuan

python/tools/mdl2fluid/float2halffloat.py

@@ -0,0 +1,70 @@
+# encoding:utf-8
+import math
+import re
+
+
+def Real2HalfFloat(data):
+    MINNUM = -65536
+    MAXNUM = 65535
+    FloatVal = 0
+    if data:
+        if data < MINNUM:
+            data = MINNUM
+        if data > MAXNUM:
+            data = MAXNUM
+
+        sign = 0
+        if data < 0:
+            sign = 1
+            data = -data
+
+        exp = math.floor((math.log2(data)))
+        expout = exp + 16
+
+        Mantial = round(data / pow(2, exp - 10)) - 1024
+
+        if expout <= 0:
+            FloatVal = 0
+        else:
+            FloatVal = sign * 32768 + expout * 1024 + Mantial
+    return FloatVal
+
+
+def ReadCfloatData(sourcefile):
+    input = []
+    with open(sourcfile, 'r') as f:
+        for line in f.readlines():
+            line = line.strip()
+            line = re.sub('\s+', ' ', line)  # 两个数字间多个空格
+            input.append(line.split(' '))
+    destfile = sourcefile.replace('.dat', '')
+    destfile = destfile.replace('.txt', '')
+    destfile += 'Out.dat'
+    with open(destfile, 'w') as fw:
+        for i in range(len(input)):
+            if len(input[i]) == 2:
+                real = Real2HalfFloat(float(input[i][0]))
+                imag = Real2HalfFloat(float(input[i][1]))
+                result = real * 65536 + imag
+                if imag and not real:
+                    fw.write('0x0000' + "%X" % result + '\n')
+                elif not imag and not real:
+                    fw.write('0x00000000' + '\n')
+                else:
+                    fw.write('0x' + "%X" % result + '\n')
+            elif len(input[i]) == 1:
+                result = Real2HalfFloat(float(input[i][0]))
+                if result:
+                    fw.write('0x' + "%X" % result + '\n')
+                else:
+                    fw.write('0x0000' + '\n')
+
+
+if __name__ == '__main__':
+    print('Tips: Input number 0 if you want to exit!\n')
+    while True:
+        sourcfile = input("input source file:\n")
+        if sourcfile is '0':
+            break
+        ReadCfloatData(sourcfile)
+        print('Transfer Success!')

python/tools/mdl2fluid/framework.proto

@@ -0,0 +1,176 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+syntax = "proto2";
+option optimize_for = LITE_RUNTIME;
+package paddle_mobile.framework.proto;
+
+enum AttrType {
+  INT = 0;
+  FLOAT = 1;
+  STRING = 2;
+  INTS = 3;
+  FLOATS = 4;
+  STRINGS = 5;
+  BOOLEAN = 6;
+  BOOLEANS = 7;
+  BLOCK = 8;
+  LONG = 9;
+}
+
+// OpDesc describes an instance of a C++ framework::OperatorBase
+// derived class type.
+message OpDesc {
+
+  message Attr {
+    required string name = 1;
+    required AttrType type = 2;
+    optional int32 i = 3;
+    optional float f = 4;
+    optional string s = 5;
+    repeated int32 ints = 6;
+    repeated float floats = 7;
+    repeated string strings = 8;
+    optional bool b = 10;
+    repeated bool bools = 11;
+    optional int32 block_idx = 12;
+    optional int64 l = 13;
+  };
+
+  message Var {
+    required string parameter = 1;
+    repeated string arguments = 2;
+  };
+
+  required string type = 3;
+  repeated Var inputs = 1;
+  repeated Var outputs = 2;
+  repeated Attr attrs = 4;
+  optional bool is_target = 5 [ default = false ];
+};
+
+// OpProto describes a C++ framework::OperatorBase derived class.
+message OpProto {
+
+  // VarProto describes the C++ type framework::Variable.
+  message Var {
+    required string name = 1;
+    required string comment = 2;
+
+    optional bool duplicable = 3 [ default = false ];
+    optional bool intermediate = 4 [ default = false ];
+    optional bool dispensable = 5 [ default = false ];
+  }
+
+  // AttrProto describes the C++ type Attribute.
+  message Attr {
+    required string name = 1;
+    required AttrType type = 2;
+    required string comment = 3;
+    // If that attribute is generated, it means the Paddle third
+    // language binding has responsibility to fill that
+    // attribute. End-User should not set that attribute.
+    optional bool generated = 4 [ default = false ];
+  }
+
+  required string type = 1;
+  repeated Var inputs = 2;
+  repeated Var outputs = 3;
+  repeated Attr attrs = 4;
+  required string comment = 5;
+}
+
+message VarType {
+  enum Type {
+    // Pod Types
+    BOOL = 0;
+    INT16 = 1;
+    INT32 = 2;
+    INT64 = 3;
+    FP16 = 4;
+    FP32 = 5;
+    FP64 = 6;
+
+    // Other types that may need additional descriptions
+    LOD_TENSOR = 7;
+    SELECTED_ROWS = 8;
+    FEED_MINIBATCH = 9;
+    FETCH_LIST = 10;
+    STEP_SCOPES = 11;
+    LOD_RANK_TABLE = 12;
+    LOD_TENSOR_ARRAY = 13;
+    PLACE_LIST = 14;
+    READER = 15;
+    CHANNEL = 16;
+    // Any runtime decided variable type is raw
+    // raw variables should manage their own allocations
+    // in operators like nccl_op
+    RAW = 17;
+    TUPLE = 18;
+  }
+
+  required Type type = 1;
+
+  message TensorDesc {
+    // Should only be PODType. Is enforced in C++
+    required Type data_type = 1;
+    repeated int64 dims = 2; // [UNK, 640, 480] is saved as [-1, 640, 480]
+  }
+  optional TensorDesc selected_rows = 2;
+
+  message LoDTensorDesc {
+    required TensorDesc tensor = 1;
+    optional int32 lod_level = 2 [ default = 0 ];
+  }
+  optional LoDTensorDesc lod_tensor = 3;
+
+  message LoDTensorArrayDesc {
+    required TensorDesc tensor = 1;
+    optional int32 lod_level = 2 [ default = 0 ];
+  }
+  optional LoDTensorArrayDesc tensor_array = 4;
+
+  message ReaderDesc { repeated LoDTensorDesc lod_tensor = 1; }
+  optional ReaderDesc reader = 5;
+
+  message ChannelDesc {
+    required Type data_type = 1;
+    required int64 capacity = 2;
+  }
+  optional ChannelDesc channel = 6;
+
+  message Tuple { repeated Type element_type = 1; }
+  optional Tuple tuple = 7;
+}
+
+message VarDesc {
+  required string name = 1;
+  required VarType type = 2;
+  optional bool persistable = 3 [ default = false ];
+}
+
+message BlockDesc {
+  required int32 idx = 1;
+  required int32 parent_idx = 2;
+  repeated VarDesc vars = 3;
+  repeated OpDesc ops = 4;
+  optional int32 forward_block_idx = 5 [ default = -1 ];
+}
+
+// Please refer to
+// https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/program.md
+// for more details.
+// TODO(panyx0718): A model can have multiple programs. Need a
+// way to distinguish them. Maybe ID or name?
+message ProgramDesc { repeated BlockDesc blocks = 1; }