From 6d8a3f83bff3da6a3f1cd821b78359c4828753e7 Mon Sep 17 00:00:00 2001 From: jwilson232 Date: Wed, 5 Aug 2026 12:31:16 +0100 Subject: [PATCH] cove: handles missing max_row property xlsx google sheet files are missing the max_row field. This commit uses the calculate_dimension function from openpyxl to determine the max_row when a file is missing it. --- .gitignore | 2 ++ .python-version | 1 + .../fixtures/badfile_missing_max_row.xlsx | Bin 0 -> 6077 bytes cove/cove_360/tests/test_browser.py | 3 +++ cove/cove_360/views.py | 11 ++++++----- 5 files changed, 12 insertions(+), 5 deletions(-) create mode 100644 .python-version create mode 100644 cove/cove_360/fixtures/badfile_missing_max_row.xlsx diff --git a/.gitignore b/.gitignore index 9b847b28..1366b037 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ *~ .ve .ve2 +.DS_Store *.pyc __pycache__ .pytest_cache/ @@ -18,6 +19,7 @@ geckodriver.log ghostdriver.log cove/lib/org-ids.json cove/lib/org-ids.json.lock +lib360dataquality.egg-info/ chromedriver/ src/ node_modules/ diff --git a/.python-version b/.python-version new file mode 100644 index 00000000..e4fba218 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/cove/cove_360/fixtures/badfile_missing_max_row.xlsx b/cove/cove_360/fixtures/badfile_missing_max_row.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..023fcd6870f6bc0353552e05a500deca2705f841 GIT binary patch literal 6077 zcmai2Wk6Kx)`p>m9zeRgq(e{|L1ZMQ92yy#0qO1*lnw!rW+(|MX^@dnT9Iz>(28{3 z0q_0fc<=FB6MOF;@7nLX;#uoi>R>eV8z?{^5XFib!w}_~P$2I;?D))`Ox>+vmd^jV z5#aT(x62$fgmv)aVeOD?5NOX>Lnuh(Zx{`#@)OLl!l>h(_Y&kz`3)(HT6?3?+4o5w zZGVKh)E?e7Bk^Hgt0KnmB7O7CkeOiHp*o2}ENILm_`RY*Y!~e04W&rgKvN;prngaH z6_IarCq!9C^7fn{-q(>g_)820@wegF{rBB;`b*XnzRLh#c)zR9Vp~{iM|**`xO=WL z7E&vMb0C{Zb1cRfLS}Z1abPi53dbtfkW@cKMfe8^MFGM^ley~GImeac&LptK&Z+jM zyER*SKN)vV-*}HotJZn5LEfu7ypqkRB>}4Fz7<$!g+?3L*Ih4mxHw(O)gxNN!EIM* zG*kAWnyJoM9gGFUGP4fGKt(~pLPJ4O|Nl(s?%-tWYz2k7IP+ayews2x-?^CoMyP-L z5h30-DXMizuG=$6Ni)QIo4QMnGEfYA=KL(uinDYBYqd%CaKHCuP4$fzn@05khFnW3yC2`-feI zDsEDK(A((4F|N2PndU@v0)I<7n`{7FQLD-ktIDw18;v^S7dXJde-e$ErP>*==TQ;; zD1J);{pqPz4WH$_+<8$KwF)l*T03i2K~lhbEY?r5CY35}Rbl;L{H}HZ+OC#VK5}+t zr|h?fIJx}O7QV}ZCCLpp?k~Q25?@zYVqJ7bT_ScIc*||GzC}c6U8i~U46%Q%)>96t z#n%)uK3F)cJ!R01dY;h3?j**vX~8B=yvb%U!3+F!*0xQ_0o%?JdZ z9E3JBgx&bS#kGTzL^9K{?_Iv?XzyKP_9nq6`6Pu!58O-lq6J5gdV*-Vvbn`8%|I%i>{Px&&cQrDj-Gt6yK2L?25 zqXV}5K*0Kw`wXpH5U%H*9GMb{&y3LA+=1vDTg1RG)6AfX6U}=D6^&{PCI_T}0n@>f z3mbkF`x-1ZZ5Ots4eg=&2KCHZrsfBFyE3CPa*F5xX74HHt@ciwdNE$9TFLh%UJ(*R z>1~6^w}+j}G4TJ;rG?A#IiLgN{nWf7xaQ5rl03=GLIKQn?T^ipKn zw&;-WOSzqc5h24b!<*5A%8Q3h6pcH?bMBtKQD(F#d*Y4`*qkK+(u86G0?+TE1IB$_ za5mC85Xi_3!lMi{=?0?9uh=l|?ilxSH7WMj z;WKm}4dn>_l#0whQSDkP8aqMlt|uh#<@2ZbcGXUPJek%2{|Fy;>+lc|q!E{4vkfDL zFoL))=Ed!}g!0MR1X|J5H)vSAH1>T11Ic6Qol9MG;tewRqg8-F^B~vJTRRQjjsAeR zDz*||2D`ZkY|+i>W3=Y}e0F?ZHVuYEDhj7(1InyIb9o7PL~yOQZ3+FXOGXOziqp9e z?N7N2S$I~jQ_XCt=gP`tH=YsscM=$`;>Hqtt&fIr!ffO9OEp~iqkp>QFgw%ERsR!R)Qkf*|ywpPY^ zAQ0J-Y43u_>7Mh2Cu(49YG~?8bm=?^(T>gTU&OD?G? zE?S58DyvlC0`-EI4^=?0jDmo{q%+z1Oi3kX5fwp$!)wX0y*nPkv$Qr14dW0)zq&4O zw5D}KKA*ba14xrJ@w%PmIBnDHf@PjPPA5?OsH6V)FjHBPr!yC=@-4R{xkryIclTpOe|VAMJ`8w$ z-lTz8wl(AZ zQ^)lS$)Df%(X5fD-6g_a3dzqbb|CHfI(93l@4lZ_4&nVn;MUi;i#9He^C?~uiUK*& zalD5xyDibD&FjTdQOtZ2X93v?LF>=Vj8;euo>Pk}(m<@GxrCED8a@19X5T;TPdk{%u5R2|tQp76=%@vX1h z5c4MPz0w1<$)2hstjk1$<_4N_Ac3mvk3e4X4VF=EA-=Bp4|r6)*%96K&E-`e)oeZ=alyG zryO%>)}2W-m`}KI0(u@S8bz*(i3x5LKfF-Ida`K12Uyca zv3PiDn4(87g{+U_g(Mk7DPz}M9K^@8AhD_v$F+9~SxNCmiLD-yNyuB#3b`tv(&xY) zA;8bX+hUAl>Dfy2CU91mM}nw=(=0}r(h6GVYE9^rLLIVIKUnf@6H6w+DmF@LCWjlp zJ1V-5L*M{lG%a6oG96F3f9l#nz3tM57?XBRo)hj^%tyFs!}MpOn9+~8T&#Vjay&w3 zkt26r&fn#*9+NU<*09D_q%L+&_V#sz-#O&H>x{MnDHA3QGL1P3Pet4=P}ONZ z4EWxvp%OB{q57&#R>AW1?GIT&8mg8rjSI;$!~4#b^K}L!-p>=}u8|~Q5i2ZZZ#j*6 zlr({Krzq<4;$?%MwDWJ%?Ai(n&$v|bRF2$G#PDuc$P+&U=u(l{hOT2OsyzeS_;~$@ z+DzXbpBa7h@jMzA=8I{myrL|+MyzNw`F{6U7CPi&FMu z%SKBC_RYHJyG1#d#@wSB8@ZW8qL}h&+IB5YS7OovPb~&LcnD-$reuOWvS?>*}4oYbUXxkJ$`m0+q>#y+qGM@ibLm+P-h#GM-8N%g zjj9wX-5v)oz=*tp`1Bjq1^0d4{@y!TlUup7JZbm7n#T?x4k;y-pVBu0E+S^FLN%tr zqB|nUw~eL~>kBh)tcmsqMadoCX=u); zgs(2TZMDHMR85K42eniq;hwt=X#H&g~7mSsjIQ(CV;?V$if+g1xX6T1Gf>pV`RYKNCd4pC`HOO3fS` zY_BMGJQzu_L1Aixcka#vFXC`TD1FgNLVhRc+P;vO&EGRm7oKWKh`pO*ms5V4;UG1B z0h^`D6rKxq5@okbi$^bY)Es}P+R{2hMjap0UA8pwsJn+0Fix)4NWVI8Z?zx$^JHuS z7vgrVa0Ct>Ay_xKD`D8PlbQD{GHe1ZB^wZ-II=42n;pgthJ}0>Fs-lB0vKi1Q&fDz z{PxQ8?uk15cC_$_)Lb<242CWu-_e30JMh&Y&eu>;cR9rm0)kUne0zX65eQ9^&bJ9~ zl~gu%a);``%2eUEFYrz*V}=%&lgm~-r6pF9VcQ$}wdeGa`H7X%MWp^dPmdr54_Jf< zh0eM`ab{7I;e}o0tKp^_!;}K4F*s{Qfh@_?27Ydqc-50JSdl|)kpnkWPi4!oHNQs0 zsdM3er(FkZ;g&1RhCNexdKFD^Y$UPwkkQ2UN0LtU(+5B4z;#bt5rVS#J|rOkDO?f) z3$={ILiJ1-NV85|s?CXwP-I_kf2Vu)F))vli|A=-sW;E8xA4oom1&nA?&HDuq}W1A z7AujYvNBeQZMuF3M(k2M`0Xl=?Dr_?3wCD_nG1((!ezMk@hWBIIybgLfNy1U<&+8R zo8aBlT^h#_vHXK##me3)+xbFDbs8w!q-M};vN%iQ%)EaUyciO5kY|hskU`n(qd`tG?(0N%Gthx|$DkU7@!$4U!Al~(O3g^k2t*)yc=15Q$ ziiH0Qev)}+JMfW9%B05V-K_{z`0%qyHR7HpS1e-+6Bi@07sa9%r-_+6}fKM*2$>!P) zefnFNx(AbM))0c8)Hsp_?7)ws$9B#lJrjYrncY?r?=PX(jLcXpV4g!ZLlyEM9LVGT za|U1XB7J!WmUV=TN*`{GT!snf!WAFMz8P0w~>MG&QPL|AM z#2t<=bg>@`ZY6X{p;#ntRG4Th&uaj+NcAG8upC1Yr66fm)VfScHObr{_uQdo^%+)2 zPBk8^bk&#HiPQ;Ih3CFtLw?F2c+!IOt zRyD_2*tU;> z`?}GtA>s!YPh1ot!U3c}jyN40>^o^wRve_~9F~}mO-VpY`H`JFMYI~bg4(gTkg})$ zAzQ=51E+e&uiE8qeXxz1|*H$K3qG&v%r3 z`ojA7inF?zsVwOu{W6Pw=^IqQ4U}KA%-ziYWZRrtFWF!H@fE!QRq zzlM6hYq`#?FDDSc5~ z6#N(Q_+7*G8hzQp{Sq6zpBnzJpZk63^`dxL#D7T+k`p0!|KBD3cNNz${jz%g5*#d}`@dudbhY>&iooxBu0z8m*8LKB;=du^?+dTT?cd$Ui+ubaIHC^5 TK>8I01sD05MmE&FS3dg}t&RaF literal 0 HcmV?d00001 diff --git a/cove/cove_360/tests/test_browser.py b/cove/cove_360/tests/test_browser.py index 9b6506ea..76ce95c3 100644 --- a/cove/cove_360/tests/test_browser.py +++ b/cove/cove_360/tests/test_browser.py @@ -161,6 +161,9 @@ def server_url(request, live_server): 'than is supported by this tool (50000). Worksheets with too many rows: ' '\'grants\' (50001 rows), \'grants_2\' (50002 rows)' ], False), + ('badfile_missing_max_row.xlsx', [ + '', + ], True), ("dei_extension.xlsx", [ "do not use the 360Giving Data Standard codelists correctly.", ], True), diff --git a/cove/cove_360/views.py b/cove/cove_360/views.py index c65a0b89..dfaaacde 100644 --- a/cove/cove_360/views.py +++ b/cove/cove_360/views.py @@ -154,11 +154,12 @@ def explore_360(request, pk, template='cove_360/explore.html'): excessive_sheets = {} try: workbook = openpyxl.reader.excel.load_workbook(file_name, read_only=True) - excessive_sheets = { - sheetname: workbook[sheetname].max_row - for sheetname in workbook.sheetnames - if workbook[sheetname].max_row > settings.MAX_XLSX_ROWS - } + for sheetname in workbook.sheetnames: + ws = workbook[sheetname] + if not ws.max_row: + ws.calculate_dimension(force=True) + if ws.max_row > settings.MAX_XLSX_ROWS: + excessive_sheets[sheetname] = ws.max_row except (zipfile.BadZipFile, openpyxl.utils.exceptions.InvalidFileException): # Exceptions associated with invalid spreadsheets are passed through for cove to handle.