{"id":505,"date":"2022-07-13T10:21:20","date_gmt":"2022-07-13T10:21:20","guid":{"rendered":"https:\/\/blog.liguanxin.cn\/?p=505"},"modified":"2022-07-13T12:40:00","modified_gmt":"2022-07-13T12:40:00","slug":"%e8%ae%ba%e6%96%87%e7%ac%94%e8%ae%b0-iccv-2021-oralco-scale-conv-attentional-image-transformers","status":"publish","type":"post","link":"https:\/\/blog.liguanxin.cn\/index.php\/2022\/07\/13\/%e8%ae%ba%e6%96%87%e7%ac%94%e8%ae%b0-iccv-2021-oralco-scale-conv-attentional-image-transformers\/","title":{"rendered":"\u8bba\u6587\u7b14\u8bb0\u2014\u2014[ICCV 2021 Oral]Co-Scale Conv-Attentional Image Transformers"},"content":{"rendered":"

\u521b\u65b0\u70b9\uff1a
\n\u2460\u534f\u540c\u5c3a\u5ea6\u7684\u5377\u79ef\u6ce8\u610f\u529b\u673a\u5236(\u5e76\u884c\u3001\u4e32\u884c)
\n\u2461\u901a\u8fc7\u5377\u79ef\u5b9e\u73b0embedding\u7684\u76f8\u5bf9\u4f4d\u7f6e(\u51cf\u5c11\u8ba1\u7b97\u91cf)<\/strong><\/p>\n

\u57fa\u4e8e\u5377\u79ef\u7684\u7684\u6ce8\u610f\u529b\u6a21\u5757\u603b\u4f53\u7ed3\u6784<\/h1>\n

\u5206\u6790\u81ea\u6ce8\u610f\u529b\u673a\u5236<\/h2>\n

\u5728\u81ea\u6ce8\u610f\u529b\u8ba1\u7b97\u4e2d
\n\"\"
\nsoftmax\u903b\u8f91\u548c\u6ce8\u610f\u56feQK^T<\/span>\u5bfc\u81f4\u4e86O(N^2)<\/span>\u7684\u7a7a\u95f4\u590d\u6742\u5ea6\u548cO(N^2C)<\/span>\u7684\u65f6\u95f4\u590d\u6742\u5ea6\uff0c\u4e8e\u662f\u4f5c\u8005\u63d0\u51fa\u4e86\u4e00\u79cd\u5206\u89e3\u65b9\u6cd5\u6765\u51cf\u5c11\u8fd9\u4e9b\u590d\u6742\u5ea6\u3002
\n\"\"
\n\u5176\u4e2d\\mathcal{\\phi}(\\cdot), \\mathcal{\\psi}(\\cdot) : \\mathbb{R}^{N \\times C} \\rightarrow \\mathbb{R}^{N \\times C^{\\prime}}<\/span>\uff0c\u7a7a\u95f4\u590d\u6742\u5ea6\u51cf\u5c11\u81f3O(NC^{\\prime}+NC+CC^{\\prime})<\/span>\uff0c\u65f6\u95f4\u590d\u6742\u5ea6O(NCC^{\\prime})<\/span><\/p>\n

\u66f4\u8fdb\u4e00\u6b65\u91c7\u7528\u4ee5\u4e0b\u516c\u5f0f
\n\"\"
\n\u7a7a\u95f4\u590d\u6742\u5ea6O(NC+C^2)<\/span>\uff0c\u65f6\u95f4\u590d\u6742\u5ea6O(NCC^2)<\/span><\/p>\n

\u57fa\u4e8e\u5377\u79ef\u7684\u76f8\u5bf9\u4f4d\u7f6e\u7f16\u7801<\/h2>\n

\"\"
\n\u5176\u4e2dP\u4e3aDW\u5377\u79ef\u7684\u6743\u91cd\uff0c\u9996\u5148\u5bf9V\u8fdb\u884cDW\u5377\u79ef\uff0c\u518d\u4e0eQ\u505a\u9010\u5143\u7d20\u4e58\uff0c\u5f97\u5230\u76f8\u5bf9\u4f4d\u7f6e\u56fe\\hat{EV}<\/span>\uff0c\u518d\u4e0e\u4e00\u4e2a0\u5411\u91cfconcat\uff0c\u4ee5\u5bf9\u9f50cls token\u3002<\/p>\n

\u57fa\u4e8e\u5377\u79ef\u7684\u4f4d\u7f6e\u7f16\u7801<\/h2>\n

\u76f8\u5bf9\u4f4d\u7f6e\u7f16\u7801\u6a21\u578b\u662fQ\u548cV\u4e4b\u95f4\u57fa\u4e8e\u4f4d\u7f6e\u7684\u5c40\u90e8\u5173\u7cfb\uff0c\u800c\u4f4d\u7f6e\u7f16\u7801\u662f\u4e00\u4e2a\u5168\u5c40\u7684\u4f4d\u7f6e\u7f16\u7801\u3002\u672c\u6587\u5728\u6ce8\u610f\u529b\u673a\u5236\u524d\u589e\u52a0\u4e86\u4e00\u4e2aDW\u5377\u79ef\u6765\u5f97\u5230\u7edd\u5bf9\u4f4d\u7f6e\u7f16\u7801\u3002<\/p>\n

\u5bf9\u4e8e\u5168\u5c40\u4f4d\u7f6e\u7f16\u7801\u91c7\u7528kernel size\u4e3a3\uff0c\u5bf9\u4e8e\u4e0d\u540c\u5934\u7684\u76f8\u5bf9\u4f4d\u7f6e\u7f16\u7801\u5206\u522b\u91c7\u75283\u30015\u30017\u7684kernel size\u3002<\/p>\n

\u603b\u4f53\u7ed3\u6784<\/h2>\n

\"\"<\/p>\n

\u534f\u540c\u5c3a\u5ea6\u7684Transformer<\/h1>\n

\u4e32\u884c\u5757<\/h2>\n

\"\"
\n\u9996\u5148\u901a\u8fc7Patch Embed\u5bf9\u56fe\u50cf\u8fdb\u884c\u4e0b\u91c7\u6837\uff0c\u7136\u540eflatten\u6210\u4e00\u7cfb\u5217\u7684\u7279\u5f81token\u5e76\u589e\u52a0\u4e00\u4e2a\u7528\u4e8e\u56fe\u50cf\u5206\u7c7b\u7684cls token\u3002\u7136\u540e\u6267\u884c\u57fa\u4e8e\u5377\u79ef\u7684\u81ea\u6ce8\u610f\u529b\u6a21\u5757\u3002\u6700\u540e\u628acls token\u5206\u79bb\uff0c\u628a\u7279\u5f81token\u8fd8\u539f\u4e3a2D\u7279\u5f81\u56fe\u3002<\/p>\n

\u5e76\u884c\u5757<\/h2>\n

\"\"<\/p>\n

\u8ba9\u4e0d\u540c\u5757\u5e76\u884c\u7684\u65b9\u5f0f\u6709\u4e24\u79cd\uff1a<\/p>\n

    \n
  • \u76f4\u63a5\u8de8\u5c42\u6ce8\u610f\u529b\uff1a\u5bf9\u4e8e\u4e0d\u540c\u5c42\u7684\u5757\uff0c\u8ba9K\u548cV\u4e0b\u91c7\u6837\u6216\u4e0a\u91c7\u6837\u6765\u5339\u914d\u5176\u4ed6\u5c3a\u5ea6\u7684\u5206\u8fa8\u7387\u6765\u6267\u884c\u4ea4\u53c9\u6ce8\u610f\u529b\u3002<\/li>\n
  • \u7279\u5f81\u63d2\u503c\u6ce8\u610f\u529b\uff1a\u4e0d\u540c\u5c3a\u5ea6\u7684\u7279\u5f81\u72ec\u7acb\u8ba1\u7b97\u81ea\u6ce8\u610f\u529b\uff0c\u8ba1\u7b97\u5b8c\u4e4b\u540e\u901a\u8fc7\u53cc\u7ebf\u6027\u63d2\u503c\u6765\u5339\u914d\u5176\u4ed6\u5c3a\u5ea6\u3002<\/li>\n<\/ul>\n

    \u672c\u6587\u6700\u7ec8\u91c7\u7528\u7279\u5f81\u63d2\u503c\u6765\u8ba9\u4e0d\u540c\u5c42\u7684\u7279\u5f81\u4ea4\u4e92\u3002<\/p>\n

    \u603b\u4f53\u7ed3\u6784<\/h1>\n

    \"\"
    \nCoaT-Lite:<\/strong> \u91c7\u7528\u91d1\u5b57\u5854\u7ed3\u6784\uff0c\u4ec5\u7531\u4e32\u884c\u5757\u6784\u6210\uff0c\u901a\u8fc7\u6700\u540e\u83b7\u5f97\u7684CLS token\u6765\u8fdb\u884c\u5206\u7c7b\u4efb\u52a1\u3002
    \nCoaT:<\/strong> \u7531\u4e32\u884c\u5757\u548c\u5e76\u884c\u5757\u6784\u6210\uff0c\u4ece\u4e09\u4e2a\u5c3a\u5ea6concat cls token\u3002<\/p>\n

    \u5b9e\u9a8c\u7ed3\u679c<\/h1>\n

    \"\"
    \n\u5728\u53c2\u6570\u91cf\u66f4\u5c11\u7684\u60c5\u51b5\u4e0b\u4e0eSwin\u6301\u5e73
    \n\u540c\u7b49\u53c2\u6570\u91cf\u7684\u60c5\u51b5\u4e0b\u51c6\u786e\u5ea6\u66f4\u9ad8<\/p>\n","protected":false},"excerpt":{"rendered":"

    \u521b\u65b0\u70b9\uff1a \u2460\u534f\u540c\u5c3a\u5ea6\u7684\u5377\u79ef\u6ce8\u610f\u529b\u673a\u5236(\u5e76\u884c\u3001\u4e32\u884c) \u2461\u901a\u8fc7\u5377\u79ef\u5b9e\u73b0embedding\u7684\u76f8\u5bf9\u4f4d\u7f6e(\u51cf\u5c11\u8ba1\u7b97\u91cf) […]<\/p>\n","protected":false},"author":1,"featured_media":510,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":[],"categories":[6],"tags":[14,13,17,11,12],"_links":{"self":[{"href":"https:\/\/blog.liguanxin.cn\/index.php\/wp-json\/wp\/v2\/posts\/505"}],"collection":[{"href":"https:\/\/blog.liguanxin.cn\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/blog.liguanxin.cn\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/blog.liguanxin.cn\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/blog.liguanxin.cn\/index.php\/wp-json\/wp\/v2\/comments?post=505"}],"version-history":[{"count":0,"href":"https:\/\/blog.liguanxin.cn\/index.php\/wp-json\/wp\/v2\/posts\/505\/revisions"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/blog.liguanxin.cn\/index.php\/wp-json\/wp\/v2\/media\/510"}],"wp:attachment":[{"href":"https:\/\/blog.liguanxin.cn\/index.php\/wp-json\/wp\/v2\/media?parent=505"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/blog.liguanxin.cn\/index.php\/wp-json\/wp\/v2\/categories?post=505"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/blog.liguanxin.cn\/index.php\/wp-json\/wp\/v2\/tags?post=505"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}