[Patchew-devel] [PATCH 2/2] views: fix Content-Transfer-Encoding and Content-Type of downloaded mbox

Paolo Bonzini pbonzini at redhat.com
Tue Nov 27 08:49:35 UTC 2018


set_payload expects the argument to be already encoded according to the
message's content type and transfer encoding.  Force a good charset
and transfer encoding, in case the source message's is incompatible
with the trailers.  Otherwise, if the source message specified for
example an iso-8859-1 encoding, we get mojibake in the utf-8 trailers.

Otherwise, adding an 8-bit trailer to a 7-bit message causes the email
module to switch to quoted-printable, without the body actually being
quoted-printable.  As a result, all equals sign in the original message
are broken.

Signed-off-by: Paolo Bonzini <pbonzini at redhat.com>
---
 tests/data/0028-tags-need-8bit-encoding.mbox.gz | Bin 0 -> 6585 bytes
 tests/test_tags.py                              |  13 +++++++++++++
 www/views.py                                    |  11 ++++++++++-
 3 files changed, 23 insertions(+), 1 deletion(-)
 create mode 100644 tests/data/0028-tags-need-8bit-encoding.mbox.gz

diff --git a/tests/data/0028-tags-need-8bit-encoding.mbox.gz b/tests/data/0028-tags-need-8bit-encoding.mbox.gz
new file mode 100644
index 0000000000000000000000000000000000000000..4156beed26b6e4f2b2f03442ddb01043d411c34f
GIT binary patch
literal 6585
zcmV;q8Aj$GiwFo|0{vS6128Z$I4yKxXLBuXWo2Y7IAUpZEoE+FZ)9n1XD)4GZ+HOQ
zT76UEN|*oJe2T8A+A3U3=+1lMaGyb56a_^<Wt_FFkaS1}@?sJY+}-cKrxQ@{GIM9V
z&r`L#mYSq<y8E18pFVHtRuToM#x>~7U(kd){4g?+AH-3T(d0{PWwuA3g(#VDwRz^z
zF!OCI^P`aO({w(|QWFJB;U)5-!18B;9R<hSfiv}ksteC>pT=%Aw$c=p7!e~dk;8~X
z6uD>;&e_<=p{YL!t&FBf#JWI;E{KvM5Di%=ifIz~Sw<a{($JyFAwp at 8#R8kr1@@~?
zG??=aUC<dnj^?3F)AB}w>Xrmw!YEu%$e0YQ#Ll at a#Z`Dik>jWN4UF$m3nmis(HDM_
z&7+r>oO!NKvw0HoJuCA}^!fK^{sTs^DPZlGhz*ZsewwAi28hx- at xSdpKOCa*>OJDz
z!Ly%vXwV(@2FQ01QDjM!RS{#X>7t4SP1ki<5j91{GC4-wC_F-<iY}uCpiOjyHB-VS
z#)!u{{4eze!^1zZ8QIp%i_#30zL_R at FeObiBmim_Rd+-<<PLzvU8!>ZC|fjCYA*c%
z$%00JutY<EE{Z9i&+!pBVH76vq~DIe@*uHpeCM&u9k@}lSB>|GPp~N(rmTbb3dZjR
zr<2Ln^1ymv0<?e^kz`W>DDVS3>jeKD!IP;qvBEDqs`kerT1ysrOfv1s;)z&D_6F3p
zQtuZ{Hx*^SXbn2{t&pdTF<^a1(wy@&wgNu2tuTbm_jd%anE*0f@<je;3cz*%PosD|
zNpwskO^D_hQyfQ#lPIG$GgPc?7Ddxyr<A_QSRhyvU{^`ngQ*fx<qmQ!Zr=2<%-^;N
zAC2i>jZ`f5pApN~8jRhrGav#s`-9qJtvPrU74 at -xQhywr)rcvH48S)u(XWB|s_4-;
zNAv3?-%LJ&rflp-6GZRzvEDtA6vGhcJc$zrtPcR-Y>Uop6qq7SP>G<)%ph?GuU)Yr
zDnIK=psT71M3w-5<*)#)>)T0`MsCI(Fe3*4X#&m&mEd0rz*iZXkKz%s{n!Ij0FRH&
zdgHA5$nd>u56 at mfKCO2d$dWEI<>L;BA~0?x;FTEr5K(eOWK;%7KZ*!bE0^lnG!;j*
zO+&|)>1Y}?C7Fs3?-2SML;aidutqGfV!tr;ui{~lk30A)0UH<XE<eb9?L46-T2#Ry
zSk)x8_|&ri!DD1s6FRe&)Tykm$H>`rg3PT~Q#ylf5hVnkylM;JG8^E)&B1*99bTAS
znx at u-^6iFMkHQQh2Y$GUso8CJoBVCDu;z&G4*2HM_N)*{2f2H%HR>L<r!ef;t6sGR
z(^I3 at 3l@v=Oe|kD#Ll&d$I*G%$mWVXYn=6G?aH;%CG9HfQ9Y%j%Vlk9*8{!WXlM0>
z6{{3yFSjy&>S222kd9Wa48C^muOw;YoM<aensmS0t*b1Wy^LS#&)2I~nADe(?&@xp
zKI!tvt&hX2WZq9Mm&3a>oZ_Ybw770}$6t->+xGosJ#I`py}S86U7b&&MRqqlKZ`Tt
z+K#U3HSH_KPo2A<6=|dB at -FBu?7NFjS5rsrDwm^%`rye!biwxg5B5ICg0fG0cloDn
zuet3Qk5GF8rlD&E0d=AUcktN{iu*4R=6P0 at k9>q~{mh!J(8c^Eq*)4T^La5X?sw^$
z&fH3zh&wp5##0ste5PJ;w at cTvqFIDaqVUBJee^jV7kAr4+|WxPz=BXS_H7e=>(z$!
zvj^loS3J;=WC6R+U!u#KP6q<Rgu3ttu|T2uF{mv+T?A;b3fuL~NNrG at rHN|SXE)yc
zO8RamZl&{F3#V#}R&08HiZA6!waVoh`j=K<OLfs_MnJ$z5Jl7_RS*?PGkC)9U#FUF
z(>UWhpw4p`!9+CVZw5ywne+V{ZnF>Gz2@>=aAu`^Ez2w$;>x0Bc-d_6^(bN65H=b#
zFZn3(Cq8UmKb-J>4A>DJOn^iL5&W9+02KDjhdIEmEPd*XS0*}Zk{&U{dJSVy7QoA~
zC!yEBY6w?1!_L*!g;2lhf>l?s1ksG8LR8~Y6D$t6oMH4EM(}L}9d2}m1xJURwVO92
zo4kQ#26^0FD#Ciy?jb)#FzIe|zYpnS5Hv_3a#>rBpja^&C`)LZ<Tn6O#taT31#vtd
z&wM*eEZ2og{-(($VQ)7M=xs#Dw<aMdw{HWhU?X=t99S9=n`i0mT${Jm at x}ODep1S=
za`G(2em_W_{A778olW%7eYa%~;^CPZg?iY!uBf+9^_5ZaE3 at y*?0(*{FTq^;?S$F@
zZqS*$+`|qXRIaZk7qh3bNaexv#F#phTc_4|X{xKF+&!OJ*3|vlZ`=K-ncS6^ls2AP
zIK3T>Chh6 at 41-~;Ou)W4w{l}y9 at Wn(X|;eLVxm(?Re0B*L14_Z$X+WLTN at u7pKNQk
zq{#}ej75W&9f$BbwPi>R995CVl4ID$4rCTU%>qReozy!fK1RL#!X5N)I!)L at B~dak
zHuS at ih9P=3l*5v&Dk7HgVM#Z1Lo^I%Q6w2unY|J<Q6z^DWDpHYSRzD{u)b*!MIF)`
z18WNFY_I_jOOV$<cr3#Ngh0!hRAK5UiELh?NLa<}#*i2RFgNJm_RV*Tnq#!3Aa6Qg
z?ykSN7XUs^7N9JjcSvyy%MGm5RfCX5U9Krj+^ic74S)H?nCXvqs$Ogs%dwf0ER_58
z-76=s2_W;~ZP(%<r_O%3G>{MiP9h*CV}fN3u*57hQ~_Ng5`{IwJ)j2ov6W+YB)^lu
z>d3MLO|lGhGGPlL60Cr+JS?$wc<l~-SlR at wGKA?WyBb(!S2i0=%JNc)6bq+9e at Rhb
z`t4^`#<H^gjNd(rn`clO7)utVoXa<*5%pYY-eyp?dSDfLkPX4Q0sa(e-2k15Cc`aA
zp{byF%K9b;W4}J_f{xahAEW)M2XQ<Fdk5~JP$R6Xx<neS6M5(5yCpW!pml3r4tib2
zLu+6H@(>B;yFfAkB at h6@!~80#3^llFa245m9WY=Xk2T6gBLUgiGtrH_6F5G20k|lV
zAr&ODuI3sdsjQ-q6Txy8y)mrZ=2p#@32;&sSWL{;nUKxP4G;QyURhf+E5lDdA$VQ2
zm`GP8wNM~vgE0y4Co<(?9aM$Qtud<gn9xftCKeTqabx7eDsnEEp=nZq0~m`O8m7!^
zzCeLDcQAy+upx}ef;Ynu^>x>Ei>l{?X{<$I(-+eggTfxr*HfwT8WjmEhOFeTRe6nP
z!4TVoN<j@~%qSLyFktk}xcX)VG=}Gz0M@<H?wjDsL9<#tlbJxc1DzxLRqf!+{6&|3
zj9R at uQ`lb1yaZ1TeLJ<wQ<H|E)Wg2n4&CT7f}ANzob(vg*o at r~{C~}(c-9sD)9Zgl
zi~|8sb(i`W?PSs)DGctZXdQFOMjVJ)OmvwCE&J;NkWp>NDIh9{6h_&bgxtY<c at QhB
zrHA`{$~v;)eZU%Gx%OS)DZTMvV71))=1KXsl1{^}a(Y6 at pkl-~x8rHJI&W6&(N*I{
zv&-`*+PimF<=OKv3~I7hxznE-%A(n<%YE&l?oE0UU7l)F(JNotM)Rp+K&H}ZXQUmr
zR^-|`9jUF0<)m(nV*C2uO*|?NugimAL{5woe5ZdM-FKGdx;DKE2YNZIL~;DRvj~!9
zYB$~UaVDwM%cf_((k>63atoHqlNzVpKyl=D7)2v#c_ofzJ!@572C_H*u6*?eU9tPJ
zlCtOf<^6s6esYgjxIsEsFO9C<SdP2y@`MhasjE*lVyi1L=|+ZeD_`8D)K&bYY=q5z
zU6X_B-bt++_LG-cI?gTz{aX9VgGi))d1*W??Th)7L(An#*>x7zH>77!wOl`GE&N{c
zY+o43O_gK#zl8v>$nhNPEinz_-K&6RmSbfWs0J8fZYSME8C_<Tcm at g&skLV%Db1>%
z{50ZqU4ca6(;InWg(>7Zd^5Bo2SPIwJ<UPK9B>GJm|5c)<#v*11W7++3RmltAWE*1
z$vgeyb`$J6XjeL<A!rwWNweoDnewm6&acLPKOS<OoXo5w at fX%C^8=87U!Em^FOgB{
zW>`P`$#857vp7Cm73FwPe}J1hKk32(SUk2hS5zGyi>l1aM1{CWb!}dk6wz`u!&Y?n
z-4HqSdBn6e*jYYlI>@XbW)-0wGYSJZl<j|59xyjak&&e<;16uTyV9VE?)=b+o>QbD
zNXd2d&7HrztR7&HS3_Bp0=A2j+ at _L(zFoiCv4 at Xk%iKC$9`PrXhC`3C+Rxs)0Ye6o
zqQDB@^h^{Wza(6bnL&pQZsuift^@yaz1b>32-}}Ja)O&MosUyH at yF2bvzKLYYE~+R
zg?%Hg1jVdiL6}WVD{n*oal{>bX0Nlz+_LBieVRjTyT$uI(gLyS{B6oL6Sc(l0OvIW
z`{uU_ii(#7v#rT1>j8QZ3#0$c#+N_Idq!vRKFXgl3{EFrcOU!?SnLJ!41Wu)Ev%YJ
z5+!N=4e<AUCs)+(O!VnZUq5~98`+4zhiHK*BF`$8pyF_gO2w8$>85`9=3W<&%(|;X
zj=S?{h|<hTn16*J1B at U{0+-PBC-WrFb6Mn(?}sBqg^9p%Ig7>S<(InpsLF^BEYJsV
z^s&wR5Rtk|9<~8KvTU2qs#1fMp#FnHydBST|9!+Sg5{C7;MYY{xP44xKb%GOGzEIR
zB(Cc+f(5iXG^b5hp>J~>*aW(aXTI%cv(<iXz#@VD&kJ8~Q*K5$TJoD)M!`qnYz0$B
zA;(=eyEn*AtkeUb0FJ_BJm$2ItWJ`Z4pEk at z<Qx9LQFb{)iRF)sv*Cwq;OABt|^Fp
zYsi5E%xaF4(*=H-vfg<<no}@S1O)v&K>^doicr5+W!&wvQ@=!e%a>fsUs6bxjInIH
zU^&OwaENZ%5_V<Pb%-rHR9AGxw#OEP^Y(^bwhT!VMc1%J%^<o=4P{Iyu^dfPY)l+V
zZAgPvg=3|1X5w8pXXzJ8wLH&rZ^n6j5p48dNJVA#1Fs2~>={Lc_i>Jp0!x$9BsYAe
z^5GE>ICMtYbBPC~c6`@GJU{U>WL5TzJ>JvE<ywX;QKy<H8;&jrx(3M^ctKG^kUOce
z%$VIfA_si#AN9)@#6#9}q at gmq5Q3m(5&6!pYFUm`s`qZdd1e&-yA9DTgzDTL!a at gk
zNzs3DJc1uE8oo`}B#q{6yWT8KXjb$r0p%fNKV?RlE9H8G{4E0gM?17YS<w0}D&?(*
zFi+m{qqz#I%^d&D!EY`Fm=}opm`zDD#G*Ww2%$nARyTTNCOPK-1mRg1Fdc?KI4-aF
zqmy4bZt?#th5Gf=BmehLjJz4BfBA8d-N%og7SR>LDp_C}1Wh+sX-w8cLy?tz$3=7#
zekJmo$3<kU{x3Q%a^A=6E4mtMJh}M2<02#%AAkI~hyp%`{2jq#55hlZ+YxS#TAW`_
z^1}g{^ZQ3FKrY3^^1uA7$j`Vh4vLUdUOWE~_tH-eiaZB|cU4W8MA2kMi}MJuK!%{h
zW)XEk)j&;ENhQB|xC8pW9qu6NUmWiEne at fM4m{<D-iM_Bvre-8{{z`y%>bJ|B_xfA
z0ZK{?{W<W!S>+7;;N$WStb%MR#?P$6Kia*8YKj)@2`VY3ZV}gXY=X_PZBWxyiEN6F
zIyN=iHvYkmLi`%(L$P4FiJxrjTp?dGw{tZnqAR9>0jAE7o|CyiWPz~_)>05v0n8?r
z1zChSch8(`j-2GjO?X&w9#$L{oIkC1d022!QPJ4X(kir7;O}tX8H_bFyXWgh{As=x
z{D+QYfH$f_z>2W|G4wG?tNC{goY!}~>XM(lRSsFSLd|uuXF^u^S|dnJwkjs*W*ARm
z^R?9Ia0jNtI?W*{<)JCe%)k#QzsAP&*=O8%w&GzldtJVotv_l*-%V4tsc6x4MXNZ|
zvAV}E^WoXo+5GD3>CN)=ZnT((PJB&0&6aK^!^?R&>e=T>U;3YRt*uE>B?&(dzoKIw
zHe#c<TduawM#lsQA_^!Mxzxr!a1+o%Yy|;-{U+MoGrec#oSE2t*@*K%AX1f;nf2vY
zSxND?GaZXavNGGI4&`Op2IQ!k)v<Ai7Bs!qHm19D-Qklq3zsL?=t!j+o?U&Ls`Fu0
z?Gv7>Z96AEbyEW`_AY*!>{q%((~A|C`$itLt+W{(YlLI~YP;pjGd1O@%HZ;eEA19L
z=g37yVdnb_g~7E8ZJhG$d^N&Ov7w!!8a28Uj7RmHH=5G6JuHc18f>;!o}dlLCvLr{
zwCTMoAD-lw`y(Hk#3!6m>yffDUoQF7kHoReGg++-URuj6W?`db>SY&1p4pDbZc;4A
zW2=skdt7$S%3H?KAgUM$&#ZCgyC=tQsO_Yl=jV|LOSwAKb7n9z_V=o_rl$+)7?SsU
zff0 at Sd8AgAyV=YKx$Nu|mZ~I32CRw7KR@?qEZ$3j*F^gB9WR at RN*6~ZVeZR;S7Zt+
zyHqpd^~=+0^wb*Oi?Pi)d~StSKJ$sxM2CAdPC=-R29aMgNuzranz;N+E6ne+#7J*L
z-GFaFygpcEee4TL{M3 at gfisXGX)ioY+9gY2IIExh=v4ayf7Q^(f^EWVd2i5cu<HD&
zSg!E;AX;g7E|V&oFE_H}zX+k@;+sVuSP&Ls`g1@<pV<(o<MT)M>u9n#(kwe<N5Bq6
z at Hm2m{gxNe<Ixkt{!UbnK16k&&j0Zy@$bZ^Up>u3eENy)Ftqx~<2=Nt*YiAg|AXT^
z=2t at b#q%<^s^{~UwLN_peKEeQU~$Y3u!GVdVa}8KK at 95nIbxoJ@t5cCt)0a3mAvli
zBYu9Z?ZNXS<o}OJTn`Sr)1d!2Qy;#Rhi3rSUr$M2>U$<zp5~dUIFv+*7bm1BeLVN`
z#L&z*V3~&z8*&f9CP^LwmZKjcGT3Zbax~g7;Rj8AbhhZJhY17w<^F>uvF<6Azfvl+
zz2q1N1MU&v<558SYQC at AyIaxcA9u&|=OOt>{_RdqqFp(ldjS*)p>Jn(zpV-LV?2Ns
z%p>`oF9S-y%j!wG$K<a~%TIj4sH at Gx=r2D!()2qYHNxPO4B1~Spv(GtaB~j*NI$j@
z%wv1^Pw--L%%gu1pMF?S|B&8&t*D#He^FH@{__yppH=lgtLpy`RdvFq*;L#5XU&%2
zN>k1DhuSthDbx8mbf>i<RT-k;K8w4MG4V1j at 0Dp>B9~1g#>}F<iP1zjgt at RKLYiuP
z&q#fGT%!W-V{e|tGCsr0Gs*=!ak;tCH7T;Vz1j=n;hIfx=&h at W!IxNeD%DF_9gk>2
zpYRoEunlBfBTH!fjymX~(V%@=X*t_!lQ#2d9fu-yj0-aLdSTGVX9P--vvb54C)Gn$
zGHFX9&Z!Q$ZI(cuqskfWF?tiY+$p5>V{{>#M4T3=@K3?aui`zJQ<%8#2>mXg^}%mm
zwUx&mp~wy!$?d_XF&WNwi&f6UCA7$;DAm(B at Cyztid~9mqi>B=b)DLG=O<*{G*w-_
zi7lZF8J#40ZTe1Ofw$@`t$Gc>7j at ujRluN1b;mUfG7f3>Z6MQJQ;s6*o$u)-^0YL7
z$p~h+A&@ukO}h&7m^qGUhg?i{X*`>AVD4U-nP)|wZhhLhM4dne@;VKRajkknhdl7*
z-nd#4((@MrxjL!H=p4Pz+`9$>l8rQ$GpxweA!2m!R|K=3VV^#ws2F#O#5uOq)RQfn
z_7|}(oF-)EByx?)-|vRX6dhuDdP*!ojlDySJo6GVe!7~W<J+?_n`0zrSv=vkAx+-*
zf_gc`2SF0vhm4Zi?iAqS?5;v&T=_0lA=f;APl#Dy7W_6;M4S`4tQVTiGyaz6kSMXf
zt6enNC-nmDG*=^$yk>Pfr2uQWZ}ad@>s=rRUWTJfnd-8{Y2Zz*@HxW2D9qB9SY2Lo
zx~sIvo at Ph{54l>SxezLF*KMBGjXsR at 1EM(yAL<O)dv(j#up_Q<C$?QcPxrpd at o|B>
zc;=iTy*aCeehzJRF~M^T|KpejV7qf}nF|I)%qj4M2BJpyJ+lm4ha%LqnZ>RxzF7Py
zlCwi7Q at h{z2AjACFKCn}cA2iP?QjzTOu)YY7Z}K*B*OQcT{GOlYzZK}t&jxxam^Bf
zzlPwiR%zKorBW at jzXLd<a}UrSpDLYkk?eGF&TW-hlM;H)9~a#Tu3KZx5(*Fs*wRZd
zdk-^xQp7c&oV$PkGJuGPy7aiPn!fE0;u_yrCas8t at fN7&0i2q`mVF=elw^&|xEq&=
zC*WxBf7PbYQa}IxI|5$zQ~k;YLZ)pqtX~vz=^Ct>v#fprXlyk~+bWDAN{uE0Y_2p!
zg6TpArZR8l0tLN~P+#d}(wi*g-!}S98%f|nSgT$9liu-1&DLmDlv?8YVusFvYou at L
zz6U-%-JwY)uBVO-Ax0vLfAJaZ&?Eoqnu$P%G~NM8zG{ZYH<F{qR~CqW)OmKg&ML5M
z3jXbJw?)o-AIWskoG{3BiGX(Av{}gdWnt|O%bUq*-+2xEX07%20bcnoU0K2&0)O-R
zToV1ruO0i5kGGL=p-i%y#@#MWz7hO~n!QRsEbMOt&ThEd7}o{l`nzA+08?AQ;5pm_
r4^GYASM#tyXJw{y#EoIEe{KMnMcDD<SN|*~{hs#Up%DCPMmqoih852Z

literal 0
HcmV?d00001

diff --git a/tests/test_tags.py b/tests/test_tags.py
index 6c162de..a49a660 100755
--- a/tests/test_tags.py
+++ b/tests/test_tags.py
@@ -12,6 +12,8 @@ import sys
 import os
 sys.path.append(os.path.dirname(__file__))
 from tests.patchewtest import PatchewTestCase, main
+import email
+from mbox import decode_payload
 
 class ImportTest(PatchewTestCase):
 
@@ -48,5 +50,16 @@ class ImportTest(PatchewTestCase):
             message = self.api_client.get(uri)
             self.assertEquals(message.data['tags'], [])
 
+    def test_mbox_with_8bit_tags(self):
+        self.cli_login()
+        self.cli_import("0028-tags-need-8bit-encoding.mbox.gz")
+        self.cli_logout()
+        mbox = self.client.get('/QEMU/20181126152836.25379-1-rkagan at virtuozzo.com/mbox')
+        parser = email.parser.BytesParser(policy=email.policy.SMTP)
+        msg = parser.parsebytes(mbox.content)
+        payload = decode_payload(msg)
+        self.assertIn('SynICState *synic = get_synic(cs);', payload)
+        self.assertIn('Reviewed-by: Philippe Mathieu-Daudé <philmd at redhat.com>', payload)
+
 if __name__ == '__main__':
     main()
diff --git a/www/views.py b/www/views.py
index 45f63c8..2aae127 100644
--- a/www/views.py
+++ b/www/views.py
@@ -17,6 +17,7 @@ from django.utils.html import format_html
 from django.conf import settings
 import api
 import email
+import quopri
 from mbox import decode_payload
 import re
 from mod import dispatch_module_hook
@@ -292,7 +293,15 @@ def view_mbox(request, project, message_id):
             return msg.as_bytes(unixfrom=True)
 
         payload = decode_payload(container)
-        container.set_payload('\n'.join(mbox_with_tags_iter(payload, m.tags)))
+        # We might be adding 8-bit trailers to a message with 7bit CTE.  For
+        # patches, quoted-printable is safe and mostly human-readable.
+        try:
+            container.replace_header('Content-Transfer-Encoding', 'quoted-printable')
+        except:
+            msg.add_header('Content-Transfer-Encoding', 'quoted-printable')
+        payload = '\n'.join(mbox_with_tags_iter(payload, m.tags))
+        payload = quopri.encodestring(payload.encode('utf-8'))
+        container.set_payload(payload, charset='utf-8')
         return msg.as_bytes(unixfrom=True)
 
     s = api.models.Message.objects.find_message(message_id, project)
-- 
2.19.1




More information about the Patchew-devel mailing list