pnnx fuse more transformer attention variants (#6371)

* pnnx fuse more transformer attention variants
* test new transformers
* always eliminate contiguous
* unified view and reshape
* better float param string

Author: nihui

.github/workflows/pnnx.yml

@@ -234,22 +234,22 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - { python: '3.8',  numpy: '1.24.4', opencv: '4.5.*',  torch: '1.8.1',  torchvision: '0.9.1',  torchaudio: '0.8.1'      }
-          - { python: '3.8',  numpy: '1.24.4', opencv: '4.5.*',  torch: '1.9.1',  torchvision: '0.10.1', torchaudio: '0.9.1'      }
-          - { python: '3.8',  numpy: '1.24.4', opencv: '4.6.*',  torch: '1.10.0', torchvision: '0.11.1', torchaudio: '0.10.0+cpu' }
-          - { python: '3.9',  numpy: '1.26.4', opencv: '4.6.*',  torch: '1.11.0', torchvision: '0.12.0', torchaudio: '0.11.0+cpu' }
-          - { python: '3.9',  numpy: '1.26.4', opencv: '4.7.*',  torch: '1.12.0', torchvision: '0.13.0', torchaudio: '0.12.0+cpu' }
-          - { python: '3.10', numpy: '1.26.4', opencv: '4.7.*',  torch: '1.13.0', torchvision: '0.14.0', torchaudio: '0.13.0+cpu' }
-          - { python: '3.10', numpy: '1.26.4', opencv: '4.8.*',  torch: '2.0.0',  torchvision: '0.15.1', torchaudio: '2.0.0+cpu'  }
-          - { python: '3.10', numpy: '1.26.4', opencv: '4.8.*',  torch: '2.1.0',  torchvision: '0.16.0', torchaudio: '2.1.0+cpu'  }
-          - { python: '3.11', numpy: '1.26.4', opencv: '4.9.*',  torch: '2.2.1',  torchvision: '0.17.1', torchaudio: '2.2.1+cpu'  }
-          - { python: '3.11', numpy: '1.26.4', opencv: '4.9.*',  torch: '2.3.0',  torchvision: '0.18.0', torchaudio: '2.3.0+cpu'  }
-          - { python: '3.11', numpy: '2.2.5',  opencv: '4.10.*', torch: '2.4.0',  torchvision: '0.19.0', torchaudio: '2.4.0+cpu'  }
-          - { python: '3.12', numpy: '2.2.5',  opencv: '4.10.*', torch: '2.5.0',  torchvision: '0.20.0', torchaudio: '2.5.0+cpu'  }
-          - { python: '3.12', numpy: '2.2.5',  opencv: '4.11.*', torch: '2.6.0',  torchvision: '0.21.0', torchaudio: '2.6.0+cpu'  }
-          - { python: '3.12', numpy: '2.2.5',  opencv: '4.11.*', torch: '2.7.0',  torchvision: '0.22.0', torchaudio: '2.7.0+cpu'  }
-          - { python: '3.13', numpy: '2.2.5',  opencv: '4.12.*', torch: '2.8.0',  torchvision: '0.23.0', torchaudio: '2.8.0+cpu'  }
-          - { python: '3.13', numpy: '2.2.5',  opencv: '4.12.*', torch: '2.9.0',  torchvision: '0.24.0', torchaudio: '2.9.0+cpu'  }
+          - { python: '3.8',  numpy: '1.24.4', opencv: '4.5.*',  torch: '1.8.1',  torchvision: '0.9.1',  torchaudio: '0.8.1',      transformers: '4.52.1' }
+          - { python: '3.8',  numpy: '1.24.4', opencv: '4.5.*',  torch: '1.9.1',  torchvision: '0.10.1', torchaudio: '0.9.1',      transformers: '4.52.1' }
+          - { python: '3.8',  numpy: '1.24.4', opencv: '4.6.*',  torch: '1.10.0', torchvision: '0.11.1', torchaudio: '0.10.0+cpu', transformers: '4.52.1' }
+          - { python: '3.9',  numpy: '1.26.4', opencv: '4.6.*',  torch: '1.11.0', torchvision: '0.12.0', torchaudio: '0.11.0+cpu', transformers: '4.52.1' }
+          - { python: '3.9',  numpy: '1.26.4', opencv: '4.7.*',  torch: '1.12.0', torchvision: '0.13.0', torchaudio: '0.12.0+cpu', transformers: '4.52.1' }
+          - { python: '3.10', numpy: '1.26.4', opencv: '4.7.*',  torch: '1.13.0', torchvision: '0.14.0', torchaudio: '0.13.0+cpu', transformers: '4.52.1' }
+          - { python: '3.10', numpy: '1.26.4', opencv: '4.8.*',  torch: '2.0.0',  torchvision: '0.15.1', torchaudio: '2.0.0+cpu',  transformers: '4.52.1' }
+          - { python: '3.10', numpy: '1.26.4', opencv: '4.8.*',  torch: '2.1.0',  torchvision: '0.16.0', torchaudio: '2.1.0+cpu',  transformers: '4.52.1' }
+          - { python: '3.11', numpy: '1.26.4', opencv: '4.9.*',  torch: '2.2.1',  torchvision: '0.17.1', torchaudio: '2.2.1+cpu',  transformers: '4.52.1' }
+          - { python: '3.11', numpy: '1.26.4', opencv: '4.9.*',  torch: '2.3.0',  torchvision: '0.18.0', torchaudio: '2.3.0+cpu',  transformers: '4.52.1' }
+          - { python: '3.11', numpy: '2.2.5',  opencv: '4.10.*', torch: '2.4.0',  torchvision: '0.19.0', torchaudio: '2.4.0+cpu',  transformers: '4.52.1' }
+          - { python: '3.12', numpy: '2.2.5',  opencv: '4.10.*', torch: '2.5.0',  torchvision: '0.20.0', torchaudio: '2.5.0+cpu',  transformers: '4.52.1' }
+          - { python: '3.12', numpy: '2.2.5',  opencv: '4.11.*', torch: '2.6.0',  torchvision: '0.21.0', torchaudio: '2.6.0+cpu',  transformers: '4.52.1' }
+          - { python: '3.12', numpy: '2.2.5',  opencv: '4.11.*', torch: '2.7.0',  torchvision: '0.22.0', torchaudio: '2.7.0+cpu',  transformers: '4.52.1' }
+          - { python: '3.13', numpy: '2.2.5',  opencv: '4.12.*', torch: '2.8.0',  torchvision: '0.23.0', torchaudio: '2.8.0+cpu',  transformers: '4.56.2' }
+          - { python: '3.13', numpy: '2.2.5',  opencv: '4.12.*', torch: '2.9.0',  torchvision: '0.24.0', torchaudio: '2.9.0+cpu',  transformers: '4.56.2' }
 
     name: test-${{ matrix.torch }}-py${{ matrix.python }}
 
@@ -319,7 +319,7 @@ jobs:
         pip3 install --user pytest wheel twine requests einops numpy==${{ matrix.numpy }} opencv-python==${{ matrix.opencv }}
         pip3 install --user torch==${{ matrix.torch }}+cpu torchvision==${{ matrix.torchvision }}+cpu torchaudio==${{ matrix.torchaudio }} --index-url https://download.pytorch.org/whl/cpu
         pip3 install --user onnx onnxscript onnxruntime
-        pip3 install --user "transformers<=4.52.1" diffusers "safetensors<=0.6.0"
+        pip3 install --user "transformers<=${{ matrix.transformers }}" diffusers "safetensors<=0.6.2"
 
     - name: setup-pytorch-execstack-or-patchelf
       if: ${{ matrix.python }} == '3.8' || ${{ matrix.python }} == '3.9'

tools/pnnx/src/CMakeLists.txt

@@ -110,6 +110,7 @@ set(pnnx_pass_level1_SRCS
 )
 
 set(pnnx_pass_level2_SRCS
+    pass_level2/eliminate_contiguous.cpp
     pass_level2/eliminate_size_numtotensor_int.cpp
     pass_level2/functionize.cpp
     pass_level2/fuse_constantlist.cpp
@@ -187,7 +188,6 @@ set(pnnx_pass_level2_SRCS
     pass_level2/F_upsample_bilinear.cpp
     pass_level2/F_upsample_nearest.cpp
     pass_level2/F_upsample.cpp
-    pass_level2/Tensor_contiguous.cpp
     pass_level2/Tensor_copy.cpp
     pass_level2/Tensor_expand.cpp
     pass_level2/Tensor_expand_as.cpp
@@ -207,7 +207,6 @@ set(pnnx_pass_level2_SRCS
     pass_level2/Tensor_slice.cpp
     pass_level2/Tensor_to.cpp
     pass_level2/Tensor_type_as.cpp
-    pass_level2/Tensor_view.cpp
     pass_level2/torch_addmm.cpp
     pass_level2/torch_amax.cpp
     pass_level2/torch_amin.cpp
@@ -362,7 +361,7 @@ set(pnnx_pass_level5_SRCS
     pass_level5/eliminate_noop_pad.cpp
     pass_level5/eliminate_noop_upsample.cpp
     pass_level5/eliminate_noop_slice.cpp
-    pass_level5/eliminate_noop_view_reshape.cpp
+    pass_level5/eliminate_noop_reshape.cpp
     pass_level5/eliminate_reshape_shape_expression.cpp
     pass_level5/eliminate_type_as.cpp
     pass_level5/eval_expression.cpp
@@ -376,7 +375,6 @@ set(pnnx_pass_level5_SRCS
     pass_level5/fuse_convtranspose1d_batchnorm1d.cpp
     pass_level5/fuse_convtranspose2d_batchnorm2d.cpp
     pass_level5/fuse_convtranspose3d_batchnorm3d.cpp
-    pass_level5/fuse_contiguous_view.cpp
     pass_level5/fuse_linear_batchnorm1d.cpp
     pass_level5/fuse_pad_conv1d.cpp
     pass_level5/fuse_pad_conv2d.cpp
@@ -583,12 +581,10 @@ set(pnnx_pass_ncnn_SRCS
     pass_ncnn/nn_UpsamplingBilinear2d.cpp
     pass_ncnn/nn_UpsamplingNearest2d.cpp
     pass_ncnn/nn_ZeroPad2d.cpp
-    pass_ncnn/Tensor_contiguous.cpp
     pass_ncnn/Tensor_permute.cpp
     pass_ncnn/Tensor_reshape.cpp
     pass_ncnn/Tensor_reshape_as.cpp
     pass_ncnn/Tensor_repeat.cpp
-    pass_ncnn/Tensor_view.cpp
     pass_ncnn/torch_addmm.cpp
     pass_ncnn/torch_amax.cpp
     pass_ncnn/torch_amin.cpp

tools/pnnx/src/ir.cpp

@@ -1529,7 +1529,8 @@ int Graph::python(const std::string& pypath, const std::string& pnnxbinpath, con
                 }
                 if (param.type == 3)
                 {
-                    fprintf(pyfp, "%f", param.f);
+                    std::string fs = float_to_string(param.f);
+                    fprintf(pyfp, "%s", fs.c_str());
                 }
                 if (param.type == 4)
                 {
@@ -1569,7 +1570,8 @@ int Graph::python(const std::string& pypath, const std::string& pnnxbinpath, con
                     fprintf(pyfp, "(");
                     for (size_t i = 0; i < param.af.size(); i++)
                     {
-                        fprintf(pyfp, "%f", param.af[i]);
+                        std::string afs = float_to_string(param.af[i]);
+                        fprintf(pyfp, "%s", afs.c_str());
                         if (i + 1 != param.af.size() || param.af.size() == 1)
                             fprintf(pyfp, ",");
                     }
@@ -1640,7 +1642,8 @@ int Graph::python(const std::string& pypath, const std::string& pnnxbinpath, con
                 }
 
                 fprintf(pyfp, "        self.%s.set_weight_bias(self_%s_weight, self_%s_bias)\n", sanitize_identifier(op->name).c_str(), sanitize_identifier(op->name).c_str(), sanitize_identifier(op->name).c_str());
-                fprintf(pyfp, "        self.%s.scale = %f\n", sanitize_identifier(op->name).c_str(), op->params.at("scale").f);
+                std::string scale_str = float_to_string(op->params.at("scale").f);
+                fprintf(pyfp, "        self.%s.scale = %s\n", sanitize_identifier(op->name).c_str(), scale_str.c_str());
                 fprintf(pyfp, "        self.%s.zero_point = %d\n", sanitize_identifier(op->name).c_str(), op->params.at("zero_point").i);
 
                 continue;
@@ -1857,9 +1860,9 @@ int Graph::python(const std::string& pypath, const std::string& pnnxbinpath, con
                 }
                 fprintf(pyfp, ")\n");
             }
-            else if (op->type == "Tensor.view" || op->type == "Tensor.reshape")
+            else if (op->type == "Tensor.reshape")
             {
-                // view reshape
+                // reshape
                 fprintf(pyfp, "v_%s = v_%s.%s(", sanitize_identifier(op->outputs[0]->name).c_str(), sanitize_identifier(op->inputs[0]->name).c_str(), op->type.substr(7).c_str());
                 if (op->inputs.size() == 2)
                 {
@@ -1887,7 +1890,7 @@ int Graph::python(const std::string& pypath, const std::string& pnnxbinpath, con
             }
             else if (op->type == "Tensor.repeat")
             {
-                // view reshape
+                // repeat
                 fprintf(pyfp, "v_%s = v_%s.%s(", sanitize_identifier(op->outputs[0]->name).c_str(), sanitize_identifier(op->inputs[0]->name).c_str(), op->type.substr(7).c_str());
                 if (op->inputs.size() == 2)
                 {
@@ -2252,7 +2255,8 @@ int Graph::python(const std::string& pypath, const std::string& pnnxbinpath, con
                     {
                         for (size_t i = 0; i < op->inputs.size(); i++)
                         {
-                            if (!op->inputnames[i].empty())
+                            bool is_input = i == 0 && op->inputnames[0] == "input";
+                            if (!op->inputnames[i].empty() && !is_input)
                                 continue;
 
                             fprintf(pyfp, "v_%s", sanitize_identifier(op->inputs[i]->name).c_str());
@@ -2262,7 +2266,8 @@ int Graph::python(const std::string& pypath, const std::string& pnnxbinpath, con
 
                         for (size_t i = 0; i < op->inputs.size(); i++)
                         {
-                            if (op->inputnames[i].empty())
+                            bool is_input = i == 0 && op->inputnames[0] == "input";
+                            if (op->inputnames[i].empty() || is_input)
                                 continue;
 
                             fprintf(pyfp, "%s=v_%s", op->inputnames[i].c_str(), sanitize_identifier(op->inputs[i]->name).c_str());
@@ -2416,10 +2421,8 @@ int Graph::python(const std::string& pypath, const std::string& pnnxbinpath, con
                         fprintf(pyfp, "(");
                         for (size_t i = 0; i < param.af.size(); i++)
                         {
-                            if (param.af[i] == (int)param.af[i])
-                                fprintf(pyfp, "%.1f", param.af[i]);
-                            else
-                                fprintf(pyfp, "%g", param.af[i]);
+                            std::string afs = float_to_string(param.af[i]);
+                            fprintf(pyfp, "%s", afs.c_str());
                             if (i + 1 != param.af.size() || param.af.size() == 1)
                                 fprintf(pyfp, ",");
                         }

tools/pnnx/src/pass_level2.cpp

@@ -9,6 +9,7 @@
 #include <unordered_map>
 #include <unordered_set>
 
+#include "pass_level2/eliminate_contiguous.h"
 #include "pass_level2/eliminate_size_numtotensor_int.h"
 #include "pass_level2/functionize.h"
 #include "pass_level2/fuse_constantlist.h"
@@ -1137,6 +1138,8 @@ void pass_level2(Graph& g)
 {
     functionize(g);
 
+    eliminate_contiguous(g);
+
     eliminate_size_numtotensor_int(g);
 
     fuse_constantlist(g);

tools/pnnx/src/pass_level2/F_hardshrink.cpp

@@ -122,4 +122,34 @@ pnnx.Output             output      1 0 out
 
 REGISTER_GLOBAL_PNNX_GRAPH_REWRITER_PASS(F_hardshrink_onnx, 100)
 
+class F_hardshrink_onnx_1 : public GraphRewriterPass
+{
+public:
+    const char* match_pattern_graph() const
+    {
+        return R"PNNXIR(7767517
+7 6
+pnnx.Input              input       0 1 input
+aten::abs               op_0        1 1 input pnnx_8
+prim::Constant          op_1        0 1 val_2 value=%lambd
+torch.le                op_2        2 1 pnnx_8 val_2 pnnx_9
+prim::Constant          op_3        0 1 scalar_tensor_default_8 value=0.0
+torch.where             op_4        3 1 pnnx_9 scalar_tensor_default_8 input out
+pnnx.Output             output      1 0 out
+)PNNXIR";
+    }
+
+    const char* type_str() const
+    {
+        return "F.hardshrink";
+    }
+
+    void write(Operator* op, const std::map<std::string, Parameter>& captured_params) const
+    {
+        op->params["lambd"] = captured_params.at("lambd");
+    }
+};
+
+REGISTER_GLOBAL_PNNX_GRAPH_REWRITER_PASS(F_hardshrink_onnx_1, 100)
+
 } // namespace pnnx

tools/pnnx/src/pass_level2/F_hardsigmoid.cpp

@@ -337,4 +337,49 @@ pnnx.Output             output      1 0 out
 
 REGISTER_GLOBAL_PNNX_GRAPH_REWRITER_PASS(F_hardsigmoid_onnx_1, 101)
 
+class F_hardsigmoid_onnx_2 : public GraphRewriterPass
+{
+public:
+    const char* match_pattern_graph() const
+    {
+        return R"PNNXIR(7767517
+7 6
+pnnx.Input              input       0 1 input
+prim::Constant          op_0        0 1 scalar_tensor_default_8 value=3.0
+aten::add               op_1        2 1 input scalar_tensor_default_8 pnnx_8
+torch.clamp             op_2        1 1 pnnx_8 pnnx_9 max=6.0 min=0.0
+prim::Constant          op_3        0 1 max_val_cast value=6.0
+aten::div               op_4        2 1 pnnx_9 max_val_cast out
+pnnx.Output             output      1 0 out
+)PNNXIR";
+    }
+
+    const char* type_str() const
+    {
+        return "F.hardsigmoid";
+    }
+};
+
+REGISTER_GLOBAL_PNNX_GRAPH_REWRITER_PASS(F_hardsigmoid_onnx_2, 100)
+
+class F_hardsigmoid_onnx_3 : public F_hardsigmoid_2_2
+{
+public:
+    const char* match_pattern_graph() const
+    {
+        return R"PNNXIR(7767517
+7 6
+pnnx.Input              input       0 1 input
+prim::Constant          op_0        0 1 scalar_tensor_default_8_pnnxshadow3 value=3.0
+aten::add               op_1        2 1 input scalar_tensor_default_8_pnnxshadow3 pnnx_16
+torch.clamp             op_2        1 1 pnnx_16 pnnx_17 max=6.0 min=0.0
+prim::Constant          op_3        0 1 val_12 value=%v1p6
+aten::mul               op_4        2 1 pnnx_17 val_12 out
+pnnx.Output             output      1 0 out
+)PNNXIR";
+    }
+};
+
+REGISTER_GLOBAL_PNNX_GRAPH_REWRITER_PASS(F_hardsigmoid_onnx_3, 100)
+
 } // namespace pnnx

tools/pnnx/src/pass_level2/F_linear.cpp

@@ -268,6 +268,41 @@ pnnx.Output             output      1 0 out
 
 REGISTER_GLOBAL_PNNX_GRAPH_REWRITER_PASS(F_linear_onnx_4, 110)
 
+class F_linear_onnx_5 : public GraphRewriterPass
+{
+public:
+    const char* match_pattern_graph() const
+    {
+        return R"PNNXIR(7767517
+5 4
+pnnx.Input              input       0 1 input
+pnnx.Attribute          weight      0 1 weight @data=(%out_features,%in_features)f32
+torch.transpose         op_0        1 1 weight t dim0=1 dim1=0
+torch.mm                op_1        2 1 input t out
+pnnx.Output             output      1 0 out
+)PNNXIR";
+    }
+
+    const char* replace_pattern_graph() const
+    {
+        return R"PNNXIR(7767517
+4 3
+pnnx.Input              input       0 1 input
+pnnx.Attribute          weight      0 1 weight
+F.linear                linear      2 1 input weight out bias=None $weight=weight
+pnnx.Output             output      1 0 out
+)PNNXIR";
+    }
+
+    void write(const std::map<std::string, Operator*>& ops, const std::map<std::string, Parameter>& captured_params, const std::map<std::string, Attribute>& captured_attrs) const
+    {
+        Operator* op_weight = ops.at("weight");
+        op_weight->attrs["data"] = captured_attrs.at("weight.data");
+    }
+};
+
+REGISTER_GLOBAL_PNNX_GRAPH_REWRITER_PASS(F_linear_onnx_5, 110)
+
 class F_linear_tnn : public GraphRewriterPass
 {
 public: