{"id":50543,"date":"2024-05-26T10:20:41","date_gmt":"2024-05-26T02:20:41","guid":{"rendered":"https:\/\/swarma.org\/?p=50543"},"modified":"2024-05-26T10:20:41","modified_gmt":"2024-05-26T02:20:41","slug":"%e4%b8%80%e4%bd%9c%e8%a7%a3%e8%af%bbicml-2024%e9%ab%98%e5%88%86%e8%ae%ba%e6%96%87-dcformer%ef%bc%9a%e5%8a%a8%e6%80%81%e7%bb%84%e5%90%88%e5%a4%9a%e5%a4%b4%e6%b3%a8%e6%84%8f%e5%8a%9b%e6%94%b9%e8%bf%9btr","status":"publish","type":"post","link":"https:\/\/swarma.org\/?p=50543","title":{"rendered":"\u4e00\u4f5c\u89e3\u8bfbICML 2024\u9ad8\u5206\u8bba\u6587 DCFormer\uff1a\u52a8\u6001\u7ec4\u5408\u591a\u5934\u6ce8\u610f\u529b\u6539\u8fdbTransformer"},"content":{"rendered":"
\n
\n
\n
\n

<\/p>\n<\/section>\n

\n
\n
\n
\n
\n
\n

<\/section>\n<\/section>\n
\n
\n

\u5bfc\u8bed<\/strong><\/p>\n<\/section>\n<\/section>\n

\n

<\/section>\n<\/section>\n<\/section>\n<\/section>\n<\/section>\n
\n
\n
\u6700\u8fd1KAN<\/a>\u7a81\u7136\u7206\u706b\uff0c\u8ba9\u4eba\u4eec\u5f00\u59cb\u610f\u8bc6\u5230\u5927\u578b\u6a21\u578b\u7684\u8ba1\u7b97\u6548\u7387\u81f3\u5173\u91cd\u8981\uff0c\u63d0\u9ad8\u5927\u578b\u6a21\u578b\u751f\u6210tokens\u7684\u901f\u5ea6\u4e5f\u662f\u81f3\u5173\u91cd\u8981\u7684\u3002\u76f8\u5bf9\u4e8e\u4e0d\u65ad\u52a0\u7801GPU\uff0c\u6539\u5584Transformer\u6a21\u578b\u67b6\u6784\u7684\u8ba1\u7b97\u6548\u7387\u662f\u4e00\u79cd\u66f4\u4e3a\u957f\u8fdc\u6709\u6548\u7684\u65b9\u6cd5\u3002\u8fd1\u671f\uff0c\u5f69\u4e91\u79d1\u6280\u5bf9Transformer\u8ba1\u7b97\u6700\u8017\u65f6\u7684\u6838\u5fc3\u7ec4\u4ef6\u2014\u2014\u591a\u5934\u6ce8\u610f\u529b\u6a21\u5757\uff08MHA\uff09\u8fdb\u884c\u4e86\u6539\u8fdb\uff0c\u4ed6\u4eec\u63a8\u51fa\u7684DCFormer\u5c06Transformer\u7684\u8ba1\u7b97\u6027\u80fd\u63d0\u5347\u67092\u500d\u4e4b\u9ad8\u3002\u76ee\u524d\uff0c\u8be5\u5de5\u4f5c\u5df2\u88abICML 2024\u4ee57\u5206\u7684\u9ad8\u5206\u5f55\u7528\u3002<\/span><\/strong><\/section>\n

<\/span><\/strong><\/section>\n
\u6211\u4eec\u9080\u8bf7\u4e86\u5f69\u4e91\u79d1\u6280\u9996\u5e2d\u79d1\u5b66\u5bb6\u8096\u8fbe\u8001\u5e08\uff0c\u4e8e\u5317\u4eac\u65f6\u95f45\u670827\u65e5\uff08\u5468\u4e00\uff0919:00-20:00\uff0c\u5728\u96c6\u667a\u201c\u540eChatGPT\u65f6\u4ee3\u201d\u8bfb\u4e66\u4f1a<\/a>\u4e2d\u6df1\u5165\u89e3\u8bfbDCFormer\u7684\u4e3b\u8981\u5de5\u4f5c\u3002\u672c\u6b21\u5206\u4eab\u5c06\u6df1\u5165\u89e3\u8bfbDCFormer\u7684\u4e3b\u8981\u5de5\u4f5c\u4ee5\u53ca\u80cc\u540e\u7684\u52a8\u673a\u548c\u7814\u7a76\u5386\u7a0b\uff0c\u5e76\u63a2\u8ba8\u6a21\u578b\u7ed3\u6784\u7814\u7a76\u7684\u672a\u6765\u53d1\u5c55\u65b9\u5411\u3002<\/span><\/strong><\/section>\n
<\/mp-common-videosnap><\/section>\n
<\/span><\/strong><\/section>\n<\/section>\n
\n
<\/strong><\/section>\n<\/section>\n
\n
<\/strong><\/strong><\/section>\n<\/section>\n<\/section>\n<\/section>\n<\/section>\n
\n
\n
\n
<\/svg><\/section>\n<\/section>\n<\/section>\n<\/section>\n<\/section>\n<\/section>\n
\n
<\/section>\n<\/section>\n

<\/section>\n

<\/section>\n
<\/section>\n

<\/span><\/section>\n
\u8fd9\u9879\u5de5\u4f5c\u7684\u8bba\u6587\u5730\u5740\u4ee5\u53ca\u5728Github\u4e0a\u5df2\u5f00\u6e90\u7684\u4ee3\u7801\u3001\u6a21\u578b\u548c\u8bad\u7ec3\u6570\u636e\u96c6\u3002<\/span><\/section>\n

<\/span><\/section>\n
\n
\n
\n
\u8bba\u6587\u6807\u9898<\/strong>\uff1aImproving Transformers with Dynamically Composable Multi-Head Attention<\/span><\/section>\n
\u8bba\u6587\u94fe\u63a5<\/strong>\uff1ahttps:\/\/arxiv.org\/abs\/2405.08553<\/em><\/span><\/section>\n
\u5f00\u6e90\u9879\u76ee\u5730\u5740<\/strong>\uff1ahttps:\/\/github.com\/Caiyun-AI\/DCFormer<\/span><\/section>\n<\/section>\n<\/section>\n<\/blockquote>\n


<\/span><\/p>\n


<\/span><\/p>\n

\n
\n
\n
\n
\n

<\/section>\n

<\/section>\n
\n
<\/svg><\/section>\n<\/section>\n<\/section>\n
\n

\u4e3b\u8bb2\u4eba<\/strong><\/span><\/strong><\/strong><\/p>\n<\/section>\n

\n

<\/section>\n

<\/section>\n<\/section>\n<\/section>\n<\/section>\n<\/section>\n<\/h3>\n


<\/p>\n

<\/p>\n

\u8096\u8fbe<\/span><\/strong>\uff0c\u5317\u4eac\u90ae\u7535\u5927\u5b66\u8bb2\u5e08\uff0c\u5f69\u4e91\u79d1\u6280\u9996\u5e2d\u79d1\u5b66\u5bb6\uff0c\u96c6\u667a\u4ff1\u4e50\u90e8\u6838\u5fc3\u6210\u5458\u3002\u957f\u671f\u4ece\u4e8b\u6df1\u5ea6\u5b66\u4e60\u3001\u5927\u6a21\u578b\u7684\u7b97\u6cd5\u7814\u7a76\u53ca\u5176\u5728\u5929\u6c14\u9884\u62a5\u3001\u673a\u5668\u7ffb\u8bd1\u3001\u673a\u5668\u8f85\u52a9\u521b\u4f5c\u3001\u7cfb\u7edf\u548c\u4ee3\u7801\u5b89\u5168\u7b49\u9886\u57df\u7684\u5e94\u7528\u7814\u7a76\uff0c\u76f8\u5173\u5de5\u4f5c\u53d1\u8868\u5728ICLR\u3001ICML\u3001TDSC\u7b49\u9876\u7ea7\u4f1a\u8bae\u548c\u671f\u520a\u3002\u76ee\u524d\u7814\u7a76\u5174\u8da3\u5305\u62ec\u5927\u6a21\u578b\u673a\u5236\u673a\u5236\u53ef\u89e3\u91ca\u6027\u3001\u9ad8\u6548\u6a21\u578b\u67b6\u6784\u8bbe\u8ba1\u3002<\/span><\/section>\n

<\/span><\/section>\n

<\/span><\/section>\n

\n
\n
\n
\n
\n

<\/section>\n

<\/section>\n
\n
<\/svg><\/section>\n<\/section>\n<\/section>\n
\n

\u5185\u5bb9\u7b80\u4ecb<\/strong><\/span><\/strong><\/strong><\/p>\n<\/section>\n

\n

<\/section>\n

<\/section>\n<\/section>\n<\/section>\n<\/section>\n<\/section>\n<\/h3>\n


<\/p>\n

\u4e0e\u5f53\u4e0b\u5728\u5927\u6a21\u578bscaling law\u6307\u5bfc\u4e0b\u7684\u5806\u7b97\u529b\u8def\u7ebf\u4e0d\u540c\uff0cDCFormer\u662f\u4e00\u9879\u5173\u4e8e\u6a21\u578b\u7ed3\u6784\u7684\u57fa\u7840\u7814\u7a76\u3002\u8be5\u5de5\u4f5c\u4eceTransformer\u6838\u5fc3\u7ec4\u4ef6\u591a\u5934\u6ce8\u610f\u529b\u6a21\u5757\uff08MHA\uff09\u5165\u624b\uff0c\u7528\u673a\u5236\u53ef\u89e3\u91ca\u6027\u65b9\u6cd5\u5206\u6790\u4e86\u5b83\u7684\u56fa\u6709\u7f3a\u9677\uff0c\u63d0\u51fa\u52a8\u6001\u7ec4\u5408\u7684\u591a\u5934\u6ce8\u610f\u529b\u673a\u5236\uff08Dynamically Composable Multi-Head Attention\uff0cDCMHA\uff09\u3002DCMHA\u65e8\u5728\u63d0\u9ad8\u6a21\u578b\u7684\u8868\u8fbe\u80fd\u529b\uff0c\u540c\u65f6\u4fdd\u6301\u53c2\u6570\u548c\u8ba1\u7b97\u6548\u7387\uff0c\u5b83\u53ef\u4ee5\u4f5c\u4e3a\u4efb\u4f55Transformer\u67b6\u6784\u4e2dMHA\u6a21\u5757\u7684\u5373\u63d2\u5373\u7528\u66ff\u4ee3\u54c1\uff0c\u4ee5\u83b7\u5f97\u76f8\u5e94\u7684DCFormer\u6a21\u578b\u3002\u5b9e\u9a8c\u8868\u660e\uff0cDCFormer\u5728\u4e0d\u540c\u7684\u6a21\u578b\u67b6\u6784\u548c\u89c4\u6a21\u4e0b\u5747\u663e\u8457\u4f18\u4e8eTransformer\uff0c\u5728\u8bed\u8a00\u5efa\u6a21\u4efb\u52a1\u4e2d\u4e0e1.7~2\u500d\u7b97\u529b\u7684\u6a21\u578b\u6027\u80fd\u76f8\u5339\u914d\u3002\u8be5\u5de5\u4f5c\u5df2\u88abICML 2024\u5f55\u7528\u3002\u672c\u6b21\u5206\u4eab\u5c06\u6df1\u5165\u89e3\u8bfbDCFormer\u7684\u4e3b\u8981\u5de5\u4f5c\u4ee5\u53ca\u80cc\u540e\u7684\u52a8\u673a\u548c\u7814\u7a76\u5386\u7a0b\uff0c\u5e76\u63a2\u8ba8\u6a21\u578b\u7ed3\u6784\u7814\u7a76\u7684\u672a\u6765\u53d1\u5c55\u65b9\u5411\u3002<\/span><\/section>\n

<\/span><\/section>\n

<\/span><\/section>\n

\n
\n
\n
\n
\n

<\/section>\n

<\/section>\n
\n
<\/svg><\/section>\n<\/section>\n<\/section>\n
\n

\u5206\u4eab\u5927\u7eb2<\/strong><\/span><\/strong><\/strong><\/p>\n<\/section>\n

\n

<\/section>\n

<\/section>\n<\/section>\n<\/section>\n<\/section>\n<\/section>\n<\/h3>\n

 
<\/p>\n