diff --git a/docs/slides/dsupdt_guide.pptx b/docs/slides/dsupdt_guide.pptx index 89e8aa0..5ac6c32 100644 Binary files a/docs/slides/dsupdt_guide.pptx and b/docs/slides/dsupdt_guide.pptx differ diff --git a/docs/slides/dsupdt_guide_ppt.pdf b/docs/slides/dsupdt_guide_ppt.pdf index c82047d..8b2792d 100644 Binary files a/docs/slides/dsupdt_guide_ppt.pdf and b/docs/slides/dsupdt_guide_ppt.pdf differ diff --git a/docs/slides/gen_dsupdt.js b/docs/slides/gen_dsupdt.js index 554645d..ac33a59 100644 --- a/docs/slides/gen_dsupdt.js +++ b/docs/slides/gen_dsupdt.js @@ -451,8 +451,8 @@ function circ(s, x, y, d, fill, glyph, gcolor, gsize) { { text:"more", options:{ fontFace:MONO, bold:true } }, { text:". Omit ", options:{} }, { text:"-DS", options:{ fontFace:MONO, bold:true } }, - { text:" on Get actions to list records across all of your datasets.", options:{} }, - ], { x:0.5, y:6.36, w:10.9, h:0.4, fontFace:SANS, fontSize:11.5, color:INK, + { text:" on Get actions to list records for all your datasets.", options:{} }, + ], { x:0.5, y:6.58, w:10.9, h:0.34, fontFace:SANS, fontSize:11, color:INK, align:"center", margin:0 }); foot(s); })(); @@ -587,9 +587,9 @@ function circ(s, x, y, d, fill, glyph, gcolor, gsize) { ["-CO","ControlOffset","delay after period end, e.g. 2D10H"], ["-CT","ControlTime","next scheduled run YYYY-MM-DD HH:NN:SS"], ["-RI","RetryInterval","wait before retry after a failure"], - ["-VI","ValidInterval","how long remote files stay valid"], + ["-VI","ValidInterval","validating window for re-checking archived files"], ["-PI","ParentIndex","run only after parent control completes"], - ["-UC","UpdateControl","preset modes: B C E F G M N O Y Z"], + ["-UC","UpdateControl","one letter per preset Mode option: A B C E F G M N O Y Z \u2014 see \u201cKey Mode Options at a Glance\u201d"], ["-MC","EMailControl","A / S / E / B / N report level"], ["-EC","ErrorControl","I ignore / Q quit / N normal"], ["-KF","KeepFile","S / R / B / N which files to keep"], @@ -1398,44 +1398,94 @@ function circ(s, x, y, d, fill, glyph, gcolor, gsize) { // ============================================================ 17 KEY MODE OPTIONS (() => { - const s = p.addSlide({ masterName:"INTERIOR" }); + const s = p.addSlide({ masterName:"INTERIOR_NG" }); kicker(s, "Reference", TEAL); title(s, "Key Mode Options at a Glance"); - const opts = [ - ["-MU","process all elapsed periods"], - ["-FU","force one update even if not due"], - ["-CP","allow current, not-yet-due period"], - ["-MO","only not-yet-archived periods"], + s.addText([ + { text:"Every mode below can be given on the command line. The left group can also be ", options:{} }, + { text:"preset per control record", options:{ bold:true, color:DEEP } }, + { text:" \u2014 each is one letter of ", options:{} }, + { text:"-UC", options:{ fontFace:MONO, bold:true, color:DEEP } }, + { text:" (UpdateControl) on the ", options:{} }, + { text:"-SC", options:{ fontFace:MONO, bold:true } }, + { text:" record.", options:{} }, + ], { x:0.5, y:1.47, w:12.3, h:0.32, fontFace:SANS, fontSize:13, color:INK, margin:0 }); + + // -UC letter -> equivalent Mode option (pg_updt.py cache_update_control) + const preset = [ + ["A","-CA","check all remote files, not just the first missing"], + ["B","-UB","use period begin time for patterns"], + ["C","-CP","allow current, not-yet-due period"], + ["E","-RE","reset end date/hour from file time"], + ["F","-FU","force one update even if not due"], + ["G","-GZ","use GMT as controlling time"], + ["M","-MU","process all elapsed periods"], + ["N","-CN","re-download only if changed on server"], + ["O","-MO","only not-yet-archived periods"], + ["Y","-NY","skip Feb 29 in leap years"], + ["Z","-VS 0","treat zero-byte files as valid"], + ]; + const cmdonly = [ ["-RA","force re-archive via dsarch"], ["-RD","re-download existing remote file"], - ["-CN","re-download if changed on server"], - ["-RE","reset end date/hour from file time"], ["-IE","skip errors, continue (with -MU)"], ["-QE","quit dataset on first error"], - ["-GZ","use GMT as controlling time"], - ["-NY","skip Feb 29 in leap years"], ["-KR","keep remote file (copy, not move)"], ["-KS","keep server file (copy, not move)"], - ["-UB","use period begin time for patterns"], ["-UT","force end/next-due time advance"], ["-BG","background; suppress screen output"], ["-NE","suppress notification email"], ]; - const y0=1.75, cw=4.05, ch=0.62, gapx=0.09, gapy=0.12; - opts.forEach((o,i)=>{ - const col=i%3, row=Math.floor(i/3); - const x=0.5+col*(cw+gapx), y=y0+row*(ch+gapy); - s.addShape(p.ShapeType.roundRect, { x, y, w:cw, h:ch, rectRadius:0.06, - fill:{color: row%2?LIGHT:TINT}, line:{color:LINE, width:1} }); - s.addShape(p.ShapeType.roundRect, { x:x+0.1, y:y+0.13, w:0.85, h:0.36, - rectRadius:0.05, fill:{color:DEEP}, line:{type:"none"} }); - s.addText(o[0], { x:x+0.1, y:y+0.13, w:0.85, h:0.36, align:"center", - valign:"middle", fontFace:MONO, bold:true, fontSize:12, color:LIGHT, margin:0 }); - s.addText(o[1], { x:x+1.05, y:y, w:cw-1.15, h:ch, fontFace:SANS, fontSize:11.5, - color:INK, margin:0, valign:"middle", lineSpacingMultiple:0.95 }); + const CW=6.05, LX=0.5, RX=6.78, HY=1.90, RY=2.34, RH=0.34, RS=0.38; + + function header(x, txt, fill) { + s.addShape(p.ShapeType.roundRect, { x, y:HY, w:CW, h:0.36, rectRadius:0.05, + fill:{color:fill}, line:{type:"none"} }); + s.addText(txt, { x:x+0.14, y:HY, w:CW-0.28, h:0.36, fontFace:SANS, bold:true, + fontSize:12, color:LIGHT, margin:0, valign:"middle" }); + } + function pill(x, y, w, txt, fill) { + s.addShape(p.ShapeType.roundRect, { x, y:y+0.02, w, h:RH-0.04, rectRadius:0.05, + fill:{color:fill}, line:{type:"none"} }); + s.addText(txt, { x, y:y+0.02, w, h:RH-0.04, align:"center", valign:"middle", + fontFace:MONO, bold:true, fontSize:11.5, color:LIGHT, margin:0 }); + } + + header(LX, "PRESETTABLE VIA -UC IN THE CONTROL RECORD", DEEP); + preset.forEach((o,i)=>{ + const y = RY + i*RS; + s.addShape(p.ShapeType.roundRect, { x:LX, y, w:CW, h:RH, rectRadius:0.05, + fill:{color: i%2?LIGHT:TINT}, line:{color:LINE, width:0.75} }); + pill(LX+0.09, y, 0.34, o[0], AMBER); + s.addText("\u2192", { x:LX+0.47, y, w:0.24, h:RH, align:"center", valign:"middle", + fontFace:SANS, bold:true, fontSize:11, color:MUTE, margin:0 }); + pill(LX+0.74, y, 0.78, o[1], DEEP); + s.addText(o[2], { x:LX+1.62, y, w:CW-1.72, h:RH, fontFace:SANS, fontSize:11, + color:INK, margin:0, valign:"middle" }); }); - s.addText("Most modes can be preset in the control record via -UC (single letters) and -KF / -MC / -EC.", { - x:0.5, y:6.32, w:10.9, h:0.3, fontFace:SANS, italic:true, fontSize:12, - color:MUTE, align:"center", margin:0 }); + + header(RX, "COMMAND LINE ONLY", TEAL); + cmdonly.forEach((o,i)=>{ + const y = RY + i*RS; + s.addShape(p.ShapeType.roundRect, { x:RX, y, w:CW, h:RH, rectRadius:0.05, + fill:{color: i%2?LIGHT:TINT}, line:{color:LINE, width:0.75} }); + pill(RX+0.09, y, 0.78, o[0], TEAL); + s.addText(o[1], { x:RX+0.97, y, w:CW-1.07, h:RH, fontFace:SANS, fontSize:11, + color:INK, margin:0, valign:"middle" }); + }); + s.addShape(p.ShapeType.roundRect, { x:RX, y:RY+9*RS+0.06, w:CW, h:2*RS-0.08, + rectRadius:0.05, fill:{color:TINT2}, line:{color:LINE, width:1} }); + s.addText([ + { text:"Other control-record presets: ", options:{ bold:true, color:DEEP } }, + { text:"-MC", options:{ fontFace:MONO, bold:true } }, + { text:" email level, ", options:{} }, + { text:"-EC", options:{ fontFace:MONO, bold:true } }, + { text:" error control, ", options:{} }, + { text:"-KF", options:{ fontFace:MONO, bold:true } }, + { text:" keep-file \u2014 set once on ", options:{} }, + { text:"-SC", options:{ fontFace:MONO, bold:true } }, + { text:" instead of on every run.", options:{} }, + ], { x:RX+0.16, y:RY+9*RS+0.06, w:CW-0.32, h:2*RS-0.08, fontFace:SANS, + fontSize:11, color:INK, margin:0, valign:"middle", lineSpacingMultiple:1.05 }); foot(s); })(); diff --git a/pyproject.toml b/pyproject.toml index 8f4b11e..901fe9a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "rda_python_dsupdt" -version = "3.0.19" +version = "3.0.20" authors = [ { name="Zaihua Ji", email="zji@ucar.edu" }, ] diff --git a/src/rda_python_dsupdt/__init__.py b/src/rda_python_dsupdt/__init__.py index c807209..a6324e8 100644 --- a/src/rda_python_dsupdt/__init__.py +++ b/src/rda_python_dsupdt/__init__.py @@ -19,7 +19,7 @@ from . import PgUpdt -__version__ = "3.0.19" +__version__ = "3.0.20" __all__ = [ "PgUpdt", diff --git a/src/rda_python_dsupdt/dsupdt.usg b/src/rda_python_dsupdt/dsupdt.usg index 77a92a8..a7de7cb 100644 --- a/src/rda_python_dsupdt/dsupdt.usg +++ b/src/rda_python_dsupdt/dsupdt.usg @@ -302,10 +302,13 @@ ControlIndex<:>Specialist<:>ParentIndex<:>Action<:>Frequency<:>ControlOffset<:>C O -(CO|ControlOffset) time offset for next process, i.e., 2D10H, 3H15N T -(CT|ControlTime) time to run dsupdt, format: YYYY-MM-DD HH:NN:SS R -(RI|RetryInterval) retry interval for failed dsupdt, i.e., 1M2D, 2H - V -(VI|ValidInterval) update valid interval, i.e., 2D, 1M - U -(UC|UpdateControl) any of B-Begin, C-Current, E-Reset End Date/Time, - F-Future, G-GMT, M-Multi, N-New, O-Miss, - Y-Skip Feb. 29, Z-Allow Zero Filesize + V -(VI|ValidInterval) validating window for re-checking archived + files, i.e., 2D, 1M + U -(UC|UpdateControl) any of A-Check All(-CA), B-Begin(-UB), + C-Current(-CP), E-Reset End Date/Time(-RE), + F-Future(-FU), G-GMT(-GZ), M-Multi(-MU), + N-New(-CN), O-Miss(-MO), Y-Skip Feb. 29(-NY), + Z-Allow Zero Filesize(-VS 0) J -(MC|EMailControl) one of A-All, E-Error, N-No, S-Summary, B-Summary when Error E -(EC|ErrorControl) one of I-Ignore Error, Q-Quit on Error, N-Neither K -(KF|KeepFile) one of S-Server File, R-Remote File, B-Keep Both, N-Neither @@ -444,7 +447,8 @@ LocalIndex<:>LocalFile<:>Action<:>ExecOrder<:>ControlIndex<:>FileArchived<:>Down J -(ED|EndDate) data end date of next update K -(EH|EndHour) data end hour of next update N -(DI|DueInterval) data next due period, i.e., 1M2D - V -(VI|ValidInterval) update valid interval, i.e., 2D, 1M + V -(VI|ValidInterval) validating window for re-checking archived + files, i.e., 2D, 1M T -(AT|AgeTime) remote file age interval, i.e., 2D, 1M W -(WD|WorkDir) working directory for processing update Z -(CL|CleanCommand) command for removing data files after update @@ -1697,8 +1701,10 @@ Information (Info) options supply values to 'dsupdt'. There are two types: -UC or -UpdateControl — a string of single letters pre-configuring Mode options in an update control record: - - B: use period begin times for temporal patterns (equivalent to -BT - mode); + - A: check all remote files instead of stopping at the first missing + one (equivalent to -CA); + - B: use period begin times for temporal patterns (equivalent to + -UB); - C: allow updates within the current period before the end time is due (equivalent to -CP); - E: reset end date/hour when the file timestamp exceeds it by one @@ -1713,11 +1719,14 @@ Information (Info) options supply values to 'dsupdt'. There are two types: - Y: skip February 29 in leap years (equivalent to -NY); - Z: treat zero-byte files as valid (equivalent to -VS 0). - -VI or -ValidInterval sets the number of days or hours that remote - files remain valid on the server. If the time difference between the - current date/time and the data date/time exceeds the interval - configured in GDEXDB by this option, 'dsupdt' will not attempt a - download. + -VI or -ValidInterval sets a validating window, in days or hours, + for re-checking files that are already archived. 'dsupdt' rolls the + data end date/hour back to the start of the window (current date/time + minus this interval) and re-checks every update period from there + through the present, re-archiving a period when a new or changed + source file has since become available. Periods older than the window + are considered final and are not re-checked. Without this option only + the next pending update period is processed. -WD or -WorkDir — the working directory for staging temporary files during download and archive operations. A root path can be set via the