From 3bc694ab88f9215b9340c97dff8a0fc8079b5cd2 Mon Sep 17 00:00:00 2001 From: Chris Piker Date: Fri, 23 Oct 2026 02:45:36 -0500 Subject: [PATCH 01/18] das_fetch: a catalog walking fetcher New utility das_fetch takes a das catalog path or URI, walks the federated catalog, translates interface settings (coords, data, formats, options, examples) into the HTTP query of a source and streams the response to stdout. Operating modes: roots, node (ls), json, url, data (get). Query keys are dotted member paths, bare members are the shortcut, and no keys at all runs example 0 for ease of testing. User files live under ~/.das: cache/nodes for fetched catalog nodes, servers for credentials. node.c: - catalog nodes load from local files as well as URLs - fetched nodes are cached with an age limit (DasNode_setCache) - new_RootNode_seed() names a node by URI out of band, either as the only root (-r URI=URL) or as a patch over the federated roots (-g URI=URL). - Nodes remember the credential manager and agent they were loaded with, and DasNode_subNode() inherits them when given NULL. - Lookup misses now log at info level. util.c: - das_rename() replaces an existing destination on Windows too http.c: - URLs with an explicit port now work New offline tests were added with 53 individual checks on Linux and MacOS, and a single check for Windows (node to URL). Developed-with: Claude Fable 5.1 --- README.md | 7 + buildfiles/Darwin.mak | 1 + buildfiles/Linux.mak | 1 + buildfiles/Windows.mak | 2 + buildfiles/cmake/das2CTests.cmake | 7 + buildfiles/sources.mak | 3 +- das3/http.c | 7 +- das3/node.c | 399 ++++- das3/node.h | 130 +- das3/util.c | 16 + das3/util.h | 14 + test/das_fetch/geomag_flex_0.7.json | 383 ++++ test/das_fetch/hsk_flex_0.7.json | 1706 ++++++++++++++++++ test/das_fetch/juno_survey_0.6.json | 348 ++++ test/das_fetch_test.sh | 217 +++ utilities/das_fetch.c | 2552 +++++++++++++++++++++++++++ 16 files changed, 5749 insertions(+), 44 deletions(-) create mode 100644 test/das_fetch/geomag_flex_0.7.json create mode 100644 test/das_fetch/hsk_flex_0.7.json create mode 100644 test/das_fetch/juno_survey_0.6.json create mode 100755 test/das_fetch_test.sh create mode 100644 utilities/das_fetch.c diff --git a/README.md b/README.md index ce12b7e..719b6c6 100644 --- a/README.md +++ b/README.md @@ -246,6 +246,13 @@ to stdout after all input is complete. All the rest: process data packet by packet and output data as soon as it's ready. +The exception at the front of a pipeline is **das_fetch**. It takes no +stream input. Give it a das catalog path and it walks the federated catalog, +works out the server URL and query string from the data source definition, +and streams the response body to standard output, whatever format that is. +Run `das_fetch node uiowa` to start browsing, or `das_fetch -h` for the +query key rules. + ## Reporting bugs Please use the issue tracker for the [das2C](https://github.com/das-developers/das2C/issues) github.com project to report any problems with the library. If you've fixed a bug, diff --git a/buildfiles/Darwin.mak b/buildfiles/Darwin.mak index d3c1bf9..3f635bf 100644 --- a/buildfiles/Darwin.mak +++ b/buildfiles/Darwin.mak @@ -397,6 +397,7 @@ test_main: $(BD) $(BD)/$(TARG).a $(BUILD_TEST_PROGS) $(BULID_UTIL_PROGS) test/das3_csv_test.sh $(BD) test/das3_info_test.sh $(BD) test/das3_from_das2_test.sh $(BD) + test/das_fetch_test.sh $(BD) # test/das_fetch/*.json nodes -> query URLs, offline @echo "INFO: Running unit test for CSVs with variable length item data, $(BD)/das3_csv" $(BD)/das3_csv < examples/ex21_tracers_cdpu_status.d3b > $(BD)/ex21_tracers_cdpu_status.csv @echo "INFO: ===============================================" diff --git a/buildfiles/Linux.mak b/buildfiles/Linux.mak index 112f674..8bf2dc3 100644 --- a/buildfiles/Linux.mak +++ b/buildfiles/Linux.mak @@ -397,6 +397,7 @@ test_main: $(BD) $(BD)/$(TARG).a $(BUILD_TEST_PROGS) $(BUILD_UTIL_PROGS) test/das3_csv_test.sh $(BD) # examples/ex23_tracers_mag_hsk.d3b -> .csv test/das3_info_test.sh $(BD) # examples/ex*.info golds for the dataset printers test/das3_from_das2_test.sh $(BD) # examples/ex11 pass-through, metafile gold, reject + test/das_fetch_test.sh $(BD) # test/das_fetch/*.json nodes -> query URLs, offline @echo "INFO: Running unit test for CSVs with variable length item data, $(BD)/das3_csv" $(BD)/das3_csv < examples/ex21_tracers_cdpu_status.d3b > $(BD)/ex21_tracers_cdpu_status.csv @echo "INFO: ===============================================" diff --git a/buildfiles/Windows.mak b/buildfiles/Windows.mak index 8b83108..581bf41 100644 --- a/buildfiles/Windows.mak +++ b/buildfiles/Windows.mak @@ -398,6 +398,8 @@ test_main: @$(BD)\TestCompat.exe $(BD) @echo INFO: Running unit test for CSVs with variable length item data, $(BD)\das3_csv $(BD)\das3_csv.exe < examples\ex21_tracers_cdpu_status.d3b > $(BD)\ex21_tracers_cdpu_status.csv + @echo INFO: Running offline catalog node to query URL check, $(BD)\das_fetch... + $(BD)\das_fetch.exe -C none url test\das_fetch\geomag_flex_0.7.json 0 @echo INFO: =============================================== @echo INFO: All core test programs completed without errors @echo INFO: =============================================== diff --git a/buildfiles/cmake/das2CTests.cmake b/buildfiles/cmake/das2CTests.cmake index 35b4ea2..7d89442 100644 --- a/buildfiles/cmake/das2CTests.cmake +++ b/buildfiles/cmake/das2CTests.cmake @@ -83,6 +83,13 @@ _das_redir(das3_csv_varlen main $ ) +# das_fetch: a saved catalog node and its example 0 become a query URL, no +# network. The full offline suite is test/das_fetch_test.sh (bash only). +_das_test(das_fetch_url main das_fetch -C none url test/das_fetch/geomag_flex_0.7.json 0) +set_tests_properties(das_fetch_url PROPERTIES PASS_REGULAR_EXPRESSION + "read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" +) + # test_spice ################################################################# if(DAS_WITH_SPICE) diff --git a/buildfiles/sources.mak b/buildfiles/sources.mak index 830a2c4..7d5c6f6 100644 --- a/buildfiles/sources.mak +++ b/buildfiles/sources.mak @@ -51,7 +51,8 @@ UTIL_SRCS = utilities/das1_inctime.c utilities/das2_prtime.c \ utilities/das1_ascii.c utilities/das1_bin_avg.c utilities/das2_bin_ratesec.c \ utilities/das2_psd.c utilities/das2_hapi.c utilities/das2_histo.c \ utilities/das2_cache_rdr.c utilities/das3_node.c utilities/das3_csv.c \ - utilities/das3_info.c utilities/das3_text.c utilities/das3_from_das2.c + utilities/das3_info.c utilities/das3_text.c utilities/das3_from_das2.c \ + utilities/das_fetch.c SPICE_UTIL_SRCS = utilities/das3_spice.c diff --git a/das3/http.c b/das3/http.c index a251713..94e52c2 100644 --- a/das3/http.c +++ b/das3/http.c @@ -318,12 +318,15 @@ bool DasHttpResp_init(DasHttpResp* pRes, const char* sUrl) pOut = sPort; int nPort; if(*pIn == ':'){ - while( (*pIn != '\0')&&(*pIn != '/')&&((pOut - sPort) < 32) ){ + ++pIn; /* The colon is a delimiter, not part of the port number. Copying + it made atoi() return 0 and every URL with an explicit port + was rejected as invalid. */ + while( (*pIn != '\0')&&(*pIn != '/')&&(*pIn != '?')&&((pOut - sPort) < 31) ){ *pOut = *pIn; ++pOut; ++pIn; } nPort = 0; if((strlen(sPort) < 1) || ((nPort = atoi(sPort)) == 0) || - (nPort > 65536) ){ + (nPort > 65535) ){ pRes->sError = das_string("Invalid port in URL %s", sUrl); return false; } diff --git a/das3/node.c b/das3/node.c index 502324a..b8a3409 100644 --- a/das3/node.c +++ b/das3/node.c @@ -19,6 +19,11 @@ #include #include #include +#include +#include +#include +#include +#include #ifndef _WIN32 #include @@ -104,6 +109,24 @@ bool DasNode_isCatalog(const DasNode* pThis) ); } +/* These four were declared in node.h since 2018 but never defined, so any + program that called them failed to link. das_fetch is the first. */ +bool DasNode_isStreamSrc(const DasNode* pThis){ + return (pThis != NULL) && (pThis->nType == d2node_stream_src); +} + +bool DasNode_isSpaseRec(const DasNode* pThis){ + return (pThis != NULL) && (pThis->nType == d2node_spase_cat); +} + +bool DasNode_isSpdfCat(const DasNode* pThis){ + return (pThis != NULL) && (pThis->nType == d2node_spdf_cat); +} + +das_node_type_e DasNode_type(const DasNode* pThis){ + return (pThis != NULL) ? pThis->nType : d2node_inv; +} + bool DasNode_isJson(const DasNode* pThis){ return ( (pThis->nType == d2node_catalog)||(pThis->nType == d2node_stream_src) @@ -149,12 +172,12 @@ static const char* DasNode_rootStr(const DasNode* pThis, const char* sStr) return NULL; } + /* Most root strings (title, description, version) are optional, so a + miss is the caller's business, not an error. Callers that require a + field check for NULL and complain themselves. */ const DasJdo* pObj = DasNode_getJdoType(pThis, das_json_type_str, sStr); - if(pObj == NULL){ - daslog_error_v("Error in node from %s, '%s' missing or not a string", - pThis->sURL, sStr); + if(pObj == NULL) return NULL; - } return ((das_json_str*)pObj->value)->string; } @@ -163,7 +186,7 @@ const char* DasNode_name(const DasNode* pThis){ return DasNode_rootStr(pThis, D2FRAG_NAME); } -static const char* DasNode_label(const DasNode* pThis){ +const char* DasNode_label(const DasNode* pThis){ const char* pLabel = DasNode_rootStr(pThis, D2FRAG_LABEL); if(pLabel != NULL) return pLabel; @@ -181,38 +204,201 @@ const char* DasNode_summary(const DasNode* pThis){ const char* DasNode_srcUrl(const DasNode* pThis){ return pThis->sURL;} +const char* DasNode_version(const DasNode* pThis){ + return DasNode_rootStr(pThis, D2FRAG_VERSION); +} + +const char* DasNode_pathSep(const DasNode* pThis){ + if(!DasNode_isCatalog(pThis)) return NULL; + return ((const DasCatNode*)pThis)->sPathSep; +} + /* ************************************************************************* */ -/* Direct Node Construction */ +/* Child listings. These read the container dictionary that was downloaded + * with the node, so they never touch the network. Linear walks of a linked + * list, fine for the few dozen entries real catalogs have. */ -/* make a node from a *specific* URL, does not call das_error functions unless - * a memory allocation error occurs because network operations are assumed to - * fail sometimes. Returns a new node allocated on the heap if the URL - * contained a recognizable DasCatalog object - */ -static DasNode* _DasNode_mkNode( - const char* sUrl, const char* sPathUri, DasCredMngr* pMgr, - const char* sAgent, float rConSec -){ - /* Read JSON bytes into the array, put a sanity limit of 20 MB on the read, - * no catalogs should ever get anywhere near that size, but you don't know */ - DasHttpResp httpRes; - DasAry* pBytesAry = das_http_readUrl( - sUrl, sAgent, pMgr, &httpRes, 1024*1024*20, rConSec - ); - if(pBytesAry == NULL){ - daslog_warn_v("%s", httpRes.sError); +size_t DasNode_subCount(const DasNode* pThis) +{ + if(!DasNode_isCatalog(pThis)) return 0; + const DasJdo* pDir = (const DasJdo*) ((const DasCatNode*)pThis)->pContainer; + if((pDir == NULL)||(pDir->type != das_json_type_dict)) return 0; + return ((const das_json_dict*)pDir->value)->length; +} + +static const das_json_dict_el* _DasNode_subEl(const DasNode* pThis, size_t uIdx) +{ + if(!DasNode_isCatalog(pThis)) return NULL; + const DasJdo* pDir = (const DasJdo*) ((const DasCatNode*)pThis)->pContainer; + if((pDir == NULL)||(pDir->type != das_json_type_dict)) return NULL; + + const das_json_dict_el* pEl = DasJdo_dictFirst(pDir); + for(size_t u = 0; (u < uIdx) && (pEl != NULL); ++u) pEl = pEl->next; + return pEl; +} + +const char* DasNode_subName(const DasNode* pThis, size_t uIdx) +{ + const das_json_dict_el* pEl = _DasNode_subEl(pThis, uIdx); + return pEl ? pEl->name->string : NULL; +} + +const DasJdo* DasNode_subStub(const DasNode* pThis, size_t uIdx) +{ + const das_json_dict_el* pEl = _DasNode_subEl(pThis, uIdx); + if((pEl == NULL)||(pEl->value == NULL)||(pEl->value->type != das_json_type_dict)) + return NULL; + return pEl->value; +} + +das_node_type_e DasNode_subType(const DasNode* pThis, size_t uIdx) +{ + const DasJdo* pStub = DasNode_subStub(pThis, uIdx); + if(pStub == NULL) return d2node_inv; + return das_node_type( DasJdo_string( DasJdo_get(pStub, D2FRAG_TYPE) ) ); +} + +/* ************************************************************************* */ +/* Local file nodes. + * + * Catalog nodes are not only served over HTTP. Test fixtures, detached + * catalogs under development, and das2py's documented 'file://' URLs all + * want a node read straight from disk. Anything that is not an http(s) URL + * is treated as a path; the "file://" prefix is accepted and stripped. + * Only the path forms "file:///abs/path" and "file://rel/path" are handled, + * host names in file URLs are not. */ + +static bool _DasNode_isFileUrl(const char* sUrl) +{ + if(sUrl == NULL) return false; + if(strncmp(sUrl, "http://", 7) == 0) return false; + if(strncmp(sUrl, "https://", 8) == 0) return false; + return true; +} + +static DasAry* _DasNode_readFile(const char* sUrl) +{ + const char* sPath = sUrl; + if(strncmp(sUrl, "file://", 7) == 0) sPath += 7; + + FILE* pFile = fopen(sPath, "rb"); + if(pFile == NULL){ + daslog_warn_v("Couldn't open node file %s, %s", sPath, strerror(errno)); return NULL; } - /* Save the final URL, may be needed for error messages, may not match - * sUrl due to re-directs */ - char sFinalUrl[256] = {'\0'}; - das_url_toStr(&(httpRes.url), sFinalUrl, 511); + DasAry* pAry = new_DasAry("node_file", vtUByte, 1, NULL, RANK_1(0), UNIT_DIMENSIONLESS); + char buf[16384]; + size_t uRead = 0; + while((uRead = fread(buf, 1, sizeof(buf), pFile)) > 0){ + if(!DasAry_append(pAry, (const ubyte*)buf, uRead)){ + dec_DasAry(pAry); + fclose(pFile); + return NULL; + } + } + bool bErr = (ferror(pFile) != 0); + fclose(pFile); + if(bErr){ + daslog_warn_v("Error reading node file %s", sPath); + dec_DasAry(pAry); + return NULL; + } + return pAry; +} +/* ************************************************************************* */ +/* Node cache */ - /* Parse the JSON data */ - size_t uBytes; - const char* pBytes = (const char*) (DasAry_getBytesIn(pBytesAry, DIM0, &uBytes)); +static char g_sNodeCacheDir[256] = {'\0'}; +static int g_nNodeCacheAge = 0; + +void DasNode_setCache(const char* sDir, int nMaxAgeSec) +{ + if((sDir == NULL)||(sDir[0] == '\0')){ + g_sNodeCacheDir[0] = '\0'; + return; + } + strncpy(g_sNodeCacheDir, sDir, sizeof(g_sNodeCacheDir) - 1); + g_sNodeCacheDir[sizeof(g_sNodeCacheDir) - 1] = '\0'; + g_nNodeCacheAge = nMaxAgeSec; +} + +/* Cache file name for a URL: the URL minus its scheme with everything + * outside [A-Za-z0-9.-] turned into '_', then a hash of the whole URL so + * that two URLs which sanitize alike still get separate files. Readable + * names make an 'ls' of the cache directory useful when debugging a + * catalog, a bare hash would not. */ +static bool _DasNode_cachePath(const char* sUrl, char* sPath, size_t uLen) +{ + if(g_sNodeCacheDir[0] == '\0') return false; + + uint32_t uHash = 2166136261u; /* FNV-1a, no need for anything stronger */ + for(const char* p = sUrl; *p != '\0'; ++p){ + uHash ^= (unsigned char)(*p); + uHash *= 16777619u; + } + + const char* p = sUrl; + if(strncmp(p, "https://", 8) == 0) p += 8; + else if(strncmp(p, "http://", 7) == 0) p += 7; + + char sName[160]; + size_t u = 0; + for(; (*p != '\0') && (u < sizeof(sName) - 1); ++p){ + char c = *p; + sName[u++] = (isalnum((unsigned char)c)||(c == '.')||(c == '-')) ? c : '_'; + } + sName[u] = '\0'; + + int n = snprintf(sPath, uLen, "%s%s%s_%08" PRIx32 ".json", g_sNodeCacheDir, DAS_DSEPS, sName, uHash); + return (n > 0) && ((size_t)n < uLen); +} + +/* A fresh enough cached copy, or NULL */ +static DasAry* _DasNode_readCache(const char* sPath) +{ + if(g_nNodeCacheAge <= 0) return NULL; + struct stat st; + if(stat(sPath, &st) != 0) return NULL; + if((time(NULL) - st.st_mtime) > g_nNodeCacheAge) return NULL; + return _DasNode_readFile(sPath); +} + +/* Written to a side file then renamed, so a reader never sees a partial + * document when two programs walk the catalog at once. Failures only cost + * the cache, so they are logged at debug level and otherwise ignored. */ +static void _DasNode_writeCache(const char* sPath, const char* pBytes, size_t uBytes) +{ + char sTmp[600]; + if(snprintf(sTmp, sizeof(sTmp), "%s.%ld.tmp", sPath, (long)time(NULL)) >= (int)sizeof(sTmp)) return; + + if(das_mkdirsto(sPath) != DAS_OKAY){ + daslog_debug_v("Couldn't create node cache directory for %s", sPath); + return; + } + FILE* pFile = fopen(sTmp, "wb"); + if(pFile == NULL){ + daslog_debug_v("Couldn't write node cache file %s, %s", sTmp, strerror(errno)); + return; + } + bool bOk = (fwrite(pBytes, 1, uBytes, pFile) == uBytes); + bOk = (fclose(pFile) == 0) && bOk; + if(!bOk || (das_rename(sTmp, sPath) != 0)){ + daslog_debug_v("Couldn't finish node cache file %s", sPath); + remove(sTmp); + } +} + +/* ************************************************************************* */ +/* Direct Node Construction */ + +/* Build a node from JSON text. sUrl is where it came from and sFinalUrl + * the same after redirects, both for messages. sPathUri may be NULL. */ +static DasNode* _DasNode_fromJson( + const char* pBytes, size_t uBytes, const char* sUrl, const char* sFinalUrl, + const char* sPathUri +){ if(uBytes < 2){ daslog_error_v("String data from %s too small to be a valid JSON document", sUrl); return NULL; @@ -224,9 +410,6 @@ static DasNode* _DasNode_mkNode( pRoot = das_json_parse_ex(pBytes, uBytes, das_jparse_flags_allow_json5, NULL, NULL, &jsonRes); - dec_DasAry(pBytesAry); /* Done with the byte array, data in json buffer now */ - DasHttpResp_freeFields(&httpRes); /* Done with the headers etc. */ - if(pRoot == NULL){ char sTmp[256] = {'\0'}; daslog_error_v("Error parsing JSON data for URL %s: %s", sFinalUrl, @@ -317,6 +500,63 @@ static DasNode* _DasNode_mkNode( return pBase; } +/* make a node from a *specific* URL, does not call das_error functions unless + * a memory allocation error occurs because network operations are assumed to + * fail sometimes. Returns a new node allocated on the heap if the URL + * contained a recognizable DasCatalog object + */ +static DasNode* _DasNode_mkNode( + const char* sUrl, const char* sPathUri, DasCredMngr* pMgr, + const char* sAgent, float rConSec +){ + /* Save the final URL, may be needed for error messages, may not match + * sUrl due to re-directs */ + char sFinalUrl[512] = {'\0'}; + + DasAry* pBytesAry = NULL; + char sCache[512] = {'\0'}; + bool bFromCache = false; + if(_DasNode_isFileUrl(sUrl)){ + pBytesAry = _DasNode_readFile(sUrl); + strncpy(sFinalUrl, sUrl, sizeof(sFinalUrl) - 1); + } + else if(_DasNode_cachePath(sUrl, sCache, sizeof(sCache)) && + ((pBytesAry = _DasNode_readCache(sCache)) != NULL)){ + daslog_debug_v("Node %s read from cache", sUrl); + strncpy(sFinalUrl, sUrl, sizeof(sFinalUrl) - 1); + bFromCache = true; + } + else{ + /* Read JSON bytes into the array, put a sanity limit of 20 MB on the + * read, no catalogs should ever get anywhere near that size, but you + * don't know */ + DasHttpResp httpRes; + pBytesAry = das_http_readUrl( + sUrl, sAgent, pMgr, &httpRes, 1024*1024*20, rConSec + ); + if(pBytesAry == NULL){ + daslog_warn_v("%s", httpRes.sError); + DasHttpResp_freeFields(&httpRes); + return NULL; + } + das_url_toStr(&(httpRes.url), sFinalUrl, sizeof(sFinalUrl)); + DasHttpResp_freeFields(&httpRes); /* Done with the headers etc. */ + } + if(pBytesAry == NULL) return NULL; + + size_t uBytes; + const char* pBytes = (const char*) (DasAry_getBytesIn(pBytesAry, DIM0, &uBytes)); + DasNode* pNode = _DasNode_fromJson(pBytes, uBytes, sUrl, sFinalUrl, sPathUri); + if(pNode != NULL){ pNode->pMgr = pMgr; pNode->sAgent = sAgent; } + + /* Only documents that parse are worth keeping, a server error page is not */ + if((pNode != NULL) && (sCache[0] != '\0') && !bFromCache) + _DasNode_writeCache(sCache, pBytes, uBytes); + + dec_DasAry(pBytesAry); /* Done with the byte array, data in json buffer now */ + return pNode; +} + /* ************************************************************************* */ /* Sub Node Construction */ @@ -551,9 +791,13 @@ static DasNode* _DasNode_loadSubNode_dasCat( } - daslog_error_v( + /* A miss at one level is normal while the walk backs out and tries other + * URLs or children, so it is not an error here. The caller knows whether + * the whole path failed and has the user's wording of it. */ + const char* sUri = DasNode_pathUri((DasNode*)pThis); + daslog_info_v( "Node %s (URI '%s') has no child node that starts with %s", - DasNode_label((DasNode*)pThis), DasNode_pathUri((DasNode*)pThis), sRelPath + DasNode_label((DasNode*)pThis), sUri ? sUri : "", sRelPath ); return NULL; @@ -576,6 +820,10 @@ static DasNode* _DasNode_loadSubNode_spaseCat( DasNode* DasNode_subNode( DasNode* pThis, const char* sRelPath, DasCredMngr* pMgr, const char* sAgent ){ + /* Credentials and agent flow down from the root unless overridden here */ + if(pMgr == NULL) pMgr = pThis->pMgr; + if(sAgent == NULL) sAgent = pThis->sAgent; + if((sRelPath == NULL) || (sRelPath[0] == '\0')){ das_error(DASERR_NODE, "Empty relative path, can't lookup scheme definition"); return NULL; @@ -607,7 +855,7 @@ DasNode* DasNode_subNode( * we don't count the null because the rel path might not be a terminating * path (i.e. it may have components below the child */ if(uLen == 0){ - daslog_warn_v("Node at %s has a zero-length sub-path", pNode->sURL); + daslog_warn_v("Node at %s has a zero-length sub-path", pThis->sURL); continue; } else{ @@ -710,6 +958,71 @@ DasNode* new_RootNode_url( return pNode; } +/* Append a JSON string literal, escaping the two characters that matter */ +static size_t _jsonStr(char* sOut, size_t uOut, const char* sIn) +{ + size_t u = 0; + if(u < uOut - 1) sOut[u++] = '"'; + for(; (*sIn != '\0') && (u < uOut - 3); ++sIn){ + if((*sIn == '"')||(*sIn == '\\')) sOut[u++] = '\\'; + sOut[u++] = *sIn; + } + if(u < uOut - 1) sOut[u++] = '"'; + sOut[u] = '\0'; + return u; +} + +DasNode* new_RootNode_seed( + const char* sUri, const char* sUrl, bool bFallThrough, DasCredMngr* pMgr, + const char* sAgent +){ + if((sUri == NULL)||(sUri[0] == '\0')||(sUrl == NULL)||(sUrl[0] == '\0')){ + daslog_error("A catalog seed needs both a URI and a URL"); + return NULL; + } + + /* The seed root is an ordinary Catalog node that only exists in memory. + * Its separator is null, like the federated root's, so child names are + * whole URI prefixes. The seed comes first, then, if asked for, a child + * with the empty name whose URLs are the federated roots. The empty name + * is a prefix of every path, so anything the seed does not claim falls + * through to the normal catalog, while everything under the seed URI is + * shadowed. Without that child the seed is all there is. */ + char sJson[4096]; + size_t u = 0; + u += snprintf(sJson + u, sizeof(sJson) - u, + "{\"type\":\"Catalog\",\"version\":\"0.5\",\"name\":\"seed\"," + "\"title\":\"Seeded catalog root\",\"separator\":null,\"catalog\":{" + ); + u += _jsonStr(sJson + u, sizeof(sJson) - u, sUri); + u += snprintf(sJson + u, sizeof(sJson) - u, ":{\"type\":\"Catalog\",\"urls\":["); + u += _jsonStr(sJson + u, sizeof(sJson) - u, sUrl); + u += snprintf(sJson + u, sizeof(sJson) - u, "]}"); + if(bFallThrough){ + u += snprintf(sJson + u, sizeof(sJson) - u, ",\"\":{\"type\":\"Catalog\",\"urls\":["); + for(int i = 0; i < g_nDasDefRoots; ++i){ + if(i > 0) u += snprintf(sJson + u, sizeof(sJson) - u, ","); + u += _jsonStr(sJson + u, sizeof(sJson) - u, g_lDasDefRoots[i]); + } + u += snprintf(sJson + u, sizeof(sJson) - u, "]}"); + } + u += snprintf(sJson + u, sizeof(sJson) - u, "}}"); + if(u >= sizeof(sJson) - 1){ + daslog_error("Catalog seed URI or URL too long"); + return NULL; + } + + /* Nothing is fetched here, the first subNode call does that and takes + the manager and agent from the root */ + DasNode* pRoot = _DasNode_fromJson(sJson, u, "memory:seed", "memory:seed", ""); + if(pRoot != NULL){ + pRoot->bIsRoot = true; + pRoot->pMgr = pMgr; + pRoot->sAgent = sAgent; + } + return pRoot; +} + DasNode* new_RootNode( const char* sPathUri, DasCredMngr* pMgr, const char* sAgent ){ @@ -717,6 +1030,17 @@ DasNode* new_RootNode( DasNode* pTop = NULL; DasNode* pNode = NULL; + /* das2py accepts "site:/..." and "test:/..." as shorthand for the two + well known children of the das2 root. Do the same here so every + client gets it, and so the two libraries agree on what a path means. */ + char sFullUri[600] = {'\0'}; + if((sPathUri != NULL) && ( + (strncmp(sPathUri, "site:", 5) == 0) || (strncmp(sPathUri, "test:", 5) == 0) + )){ + snprintf(sFullUri, sizeof(sFullUri) - 1, "%s%s", D2URI_ROOT, sPathUri); + sPathUri = sFullUri; + } + /* Changing node lookup and time outs. New strategy, try to contact a node if it doesn't work within the time out, try the next one. If all fail then increase the time out and try again. @@ -753,6 +1077,9 @@ DasNode* new_RootNode( * deleted without an explicit call to del_RootNode, then delete the rest, * pTop will not be deleted if pTop == pNode */ pNode->bIsRoot = true; + pNode->pMgr = pMgr; /* pTop was fetched without them, the kept node + hands them to whatever it loads later */ + pNode->sAgent = sAgent; del_NonRootNode(pTop); return pNode; diff --git a/das3/node.h b/das3/node.h index c3d7484..1afb21d 100644 --- a/das3/node.h +++ b/das3/node.h @@ -129,6 +129,7 @@ extern "C" { /* Defines for common document fragments */ #define D2FRAG_TYPE "type" +#define D2FRAG_VERSION "version" #define D2FRAG_NAME "name" #define D2FRAG_LABEL "label" #define D2FRAG_TITLE "title" @@ -165,6 +166,10 @@ typedef struct das_node { char sPath[512]; /* Holds the Path URI for this node */ bool bIsRoot; /* True if a local root node (i.e. is a memory manager) */ void* pDom; /* A pointer to the document tokens. */ + DasCredMngr* pMgr; /* Credentials used to load this node, inherited by + children. Borrowed, the caller keeps it alive. */ + const char* sAgent; /* User agent used to load this node, inherited by + children. Borrowed, usually a string literal. */ } DasNode; /** @} */ @@ -266,6 +271,40 @@ DAS_API DasNode* new_RootNode( DAS_API DasNode* new_RootNode_url(const char* sUrl, const char* sPathUri, DasCredMngr* pMgr, const char* sAgent); +/** Make a catalog root that names one node by URI and loads it from a URL. + * + * A seed states, out of band, that the node at sUrl is the one named sUri, + * for example that a development server's copy of a catalog branch stands + * for the branch's federated name, or that a DOI is the URI of a catalog + * kept outside the das2.org tag scheme. Paths under the seed URI load + * from the seeded URL and shadow whatever the federated catalog says for + * them. What happens to paths outside the seed URI is up to bFallThrough: + * true patches the federated catalog, so those paths resolve through the + * compiled in roots as usual; false makes the seed the only root, so they + * fail to resolve. + * + * Nothing is fetched here. The returned root is an in memory Catalog node, + * use DasNode_subNode() on it with sUri or any path to load nodes, and + * del_RootNode() to free everything loaded through it. Nodes loaded this + * way report sUri based path URIs from DasNode_pathUri(). + * + * @param sUri The catalog URI the seeded node has, such as + * "tag:das2.org,2012:site:/uiowa/tracers/tracers-1" + * @param sUrl Where to load that node from, any URL or file path that + * new_RootNode_url() accepts + * @param bFallThrough If true paths outside sUri resolve through the + * federated roots, if false the seed is the whole catalog + * @param pMgr Credentials manager for nodes loaded through this root, may + * be NULL. DasNode_subNode() inherits it unless given another. + * @param sAgent User agent string for the same, may be NULL + * @return A new root node, or NULL if the arguments are unusable + * @memberof DasNode + */ +DAS_API DasNode* new_RootNode_seed( + const char* sUri, const char* sUrl, bool bFallThrough, DasCredMngr* pMgr, + const char* sAgent +); + /** Get a das2 catalog node contained item. * * This function will search down the catalog hierarchy and find the requested @@ -283,12 +322,13 @@ DAS_API DasNode* new_RootNode_url(const char* sUrl, const char* sPathUri, * the site catalog, but "/uiowa" would not since the separator * defined for site is ":/". * - * @param sAgent The user agent string you wish to send to the server. If - * NULL then the string "libdas/3" is sent. - * * @param pMgr A credentials manager object to consult if a password is - * requested. May be set to NULL to indicate that only public - * items may be requested. + * requested. NULL means inherit the manager this node was + * loaded with, which is the one given to new_RootNode() or + * its relatives. Pass one to override it for this lookup. + * + * @param sAgent The user agent string to send. NULL inherits the agent this + * node was loaded with; if none was ever given, "libdas/3". * * @return A pointer to the child node, or NULL if no such child existed * in the node cache nor could not be downloaded. @@ -367,6 +407,86 @@ DAS_API const char* DasNode_title(const DasNode* pThis); */ DAS_API const char* DasNode_summary(const DasNode* pThis); +/** Get the node's "version" field, if provided + * + * This is the version of the node *definition* (ex: "0.5" for catalogs, + * "0.6" or "0.7" for HttpStreamSrc) not a data version. Clients need it + * because the interface layout of source nodes changed between 0.6 and 0.7. + * @returns the version string or NULL + * @memberof DasNode + */ +DAS_API const char* DasNode_version(const DasNode* pThis); + +/** Get a short label for the node, the "label" field if present, else "name" + * @returns the label, or NULL if the node has neither field + * @memberof DasNode + */ +DAS_API const char* DasNode_label(const DasNode* pThis); + +/** Get the separator a container node puts between its own path URI and + * the name of a child. + * + * @returns "/" by default, the empty string if the node's "separator" is + * null, or NULL if this is not a container node. + * @memberof DasNode + */ +DAS_API const char* DasNode_pathSep(const DasNode* pThis); + +/** Keep local copies of catalog nodes fetched over HTTP. + * + * Walking a catalog from a root to a data source takes one request per + * level. With a cache directory set, each node document is saved after it + * parses and the saved copy is used for later requests of the same URL + * while it is younger than nMaxAgeSec. Nodes read from local files are + * never cached. The setting is process wide. + * + * @param sDir Directory for the copies, created on first use. NULL or an + * empty string turns caching off (the default). + * + * @param nMaxAgeSec Copies older than this many seconds are fetched again. + * Zero or less means always fetch, but still save the result, which + * refreshes the cache along the path walked. + * + * @memberof DasNode + */ +DAS_API void DasNode_setCache(const char* sDir, int nMaxAgeSec); + +/** Number of children listed by a container node. + * + * This inspects the listing only, nothing is downloaded. Children of a + * container are known by name and type before they are loaded, just like + * directory entries. + * + * @returns the number of listed children, 0 for non-container nodes + * @memberof DasNode + */ +DAS_API size_t DasNode_subCount(const DasNode* pThis); + +/** Get the name of a listed child by index. No network activity. + * @param uIdx 0 to DasNode_subCount() - 1 + * @returns the child's name, or NULL if the index is out of range + * @memberof DasNode + */ +DAS_API const char* DasNode_subName(const DasNode* pThis, size_t uIdx); + +/** Get the listing entry (stub) for a child by index. No network activity. + * + * The stub is the dictionary the container holds for the child. It always + * has "type" and "urls", and often "label", "title" or "description". It + * is not the child node. Use DasNode_subNode() to load that. + * + * @returns the stub dictionary, or NULL if the index is out of range + * @memberof DasNode + */ +DAS_API const DasJdo* DasNode_subStub(const DasNode* pThis, size_t uIdx); + +/** Get the node type of a listed child by index. No network activity. + * @returns the type, d2node_inv if the type string is unknown to this + * library (ex: "Das2DSDF", "WebSocSrc") or the index is bad + * @memberof DasNode + */ +DAS_API das_node_type_e DasNode_subType(const DasNode* pThis, size_t uIdx); + /** If true the internal data for this node are in JSON format * diff --git a/das3/util.c b/das3/util.c index b91a141..8eba615 100644 --- a/das3/util.c +++ b/das3/util.c @@ -725,6 +725,22 @@ bool das_isfile(const char* path) else return false; } +int das_rename(const char* sFrom, const char* sTo) +{ +#ifdef _WIN32 + if(MoveFileExA(sFrom, sTo, MOVEFILE_REPLACE_EXISTING)) return 0; + /* The two causes a caller is likely to act on, the rest is EIO */ + switch(GetLastError()){ + case ERROR_FILE_NOT_FOUND: case ERROR_PATH_NOT_FOUND: errno = ENOENT; break; + case ERROR_ACCESS_DENIED: case ERROR_SHARING_VIOLATION: errno = EACCES; break; + default: errno = EIO; + } + return -1; +#else + return rename(sFrom, sTo); +#endif +} + void das_stdio_binary(FILE* pFile) { #ifdef _WIN32 diff --git a/das3/util.h b/das3/util.h index d8b7fff..655d72f 100644 --- a/das3/util.h +++ b/das3/util.h @@ -532,6 +532,20 @@ DAS_API bool das_copyfile(const char* src, const char* dest, mode_t mode); */ DAS_API bool das_isfile(const char* path); +/** Rename a file, replacing the destination if it exists. + * + * POSIX rename(2) already does this, atomically. The Windows C runtime's + * rename() fails when the destination exists, so there this is a wrapper + * around MoveFileEx() with MOVEFILE_REPLACE_EXISTING, which is the atomic + * replace within one volume. Use it for the write-to-side-file-then-rename + * pattern so that the pattern behaves the same everywhere. + * + * @param sFrom The existing file + * @param sTo The new name, any file already there is replaced + * @return 0 on success, -1 on failure with errno set as rename(2) would + */ +DAS_API int das_rename(const char* sFrom, const char* sTo); + /** Put a stdio stream in binary mode * * Das streams count bytes, but on Windows stdin and stdout start in text mode, diff --git a/test/das_fetch/geomag_flex_0.7.json b/test/das_fetch/geomag_flex_0.7.json new file mode 100644 index 0000000..792f26f --- /dev/null +++ b/test/das_fetch/geomag_flex_0.7.json @@ -0,0 +1,383 @@ +{ + "label": "GeomagCoords", + "name": "ts2_geomag_coords", + "title": "TRACERS 2 Geomagnetic Location Coordinates, Predicted", + "type": "HttpStreamSrc", + "version": "0.7", + "protocol": { + "method": "GET", + "convention": "HTTP/1.1", + "baseUrls": [ + "http://127.0.0.1:8765/data/geomag.d2s", + "http://127.0.0.1:8765/data/geomag_alt.d2s" + ], + "authorization": { + "required": false + }, + "httpParams": { + "read.time.min": { + "required": true, + "type": "isotime" + }, + "read.time.max": { + "required": true, + "type": "isotime" + }, + "read.time.inter": { + "required": true, + "type": "real", + "units": "s" + }, + "read.data": { + "type": "FlagSet", + "required": false, + "flagSep": ",", + "flags": { + "lshell": { + "value": "lshell" + } + } + }, + "format.type": { + "required": false, + "type": "enum", + "enum": [ + "das", + "cdf", + "csv" + ] + }, + "format.secfrac": { + "required": false, + "type": "integer", + "range": [ + 0, + 9 + ] + }, + "format.sigdigit": { + "required": false, + "type": "integer", + "range": [ + 2, + 17 + ] + }, + "format.delim": { + "required": false, + "type": "string" + }, + "format.headers": { + "required": false, + "type": "boolean" + }, + "format.csv.props": { + "required": false, + "type": "boolean" + }, + "format.csv.dsid": { + "required": false, + "type": "boolean" + }, + "format.cdf.nocomp": { + "required": false, + "type": "boolean" + }, + "format.serial": { + "label": "Serialization", + "required": false, + "type": "enum", + "enum": [ + "text", + "binary" + ] + }, + "format.version": { + "required": false, + "type": "string" + } + } + }, + "contacts": [ + { + "type": "maintainer", + "name": "Kevin Steele", + "email": "kevin-steele@uiowa.edu" + }, + { + "type": "scientific", + "name": "Ivar Christopher" + } + ], + "interface": { + "coords": { + "time": { + "label": "Time", + "props": { + "min": { + "label": "Minimum", + "value": null, + "title": "Minimum time value to stream", + "set": { + "param": "read.time.min", + "required": true + } + }, + "max": { + "label": "Maximum", + "value": null, + "title": "Maximum time value to stream", + "set": { + "param": "read.time.max", + "required": true + } + }, + "units": { + "value": "UTC" + }, + "inter": { + "label": "Interval", + "value": null, + "units": "s", + "title": "The time interval between output points form the SPICE model", + "set": { + "param": "read.time.inter", + "required": false + } + } + }, + "validRange": [ + "2024-01-01T00:00", + "now" + ] + } + }, + "data": { + "lshell": { + "label": "L-Shell", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "lshell" + } + } + }, + "title": "L-Shell Index" + } + }, + "examples": [ + { + "settings": { + "coords/time/props/min": "2026-01-01", + "coords/time/props/max": "2026-01-02", + "coords/time/props/inter": "60" + }, + "label": "First day of 2026" + } + ], + "formats": { + "cdf": { + "label": "CDF file", + "title": "NASA Common Data Format", + "mimeTypes": [ + "application/x-cdf" + ], + "props": { + "enabled": { + "type": "boolean", + "xorGroup": "format", + "value": false, + "set": { + "value": true, + "param": "format.type", + "pval": "cdf" + } + }, + "nocompress": { + "value": false, + "type": "boolean", + "label": "No-Compression", + "title": "Do not compress variables, needed for PDS compliant output.", + "set": { + "value": true, + "param": "format.cdf.nocomp", + "pval": "1" + } + } + } + }, + "das": { + "label": "das stream", + "title": "Streaming format for plots", + "mimeTypes": [ + "application/vnd.das2.das2stream", + "application/vnd.das.stream", + "text/vnd.das2.das2stream" + ], + "props": { + "enabled": { + "type": "boolean", + "xorGroup": "format", + "value": true, + "set": { + "value": true, + "param": "format.type", + "pval": "das" + } + }, + "version": { + "label": "Stream Version", + "value": "das2", + "type": "enum" + }, + "serial": { + "label": "Serialization", + "value": "binary", + "set": { + "param": "format.serial", + "enum": [ + { + "value": "text" + }, + { + "value": "binary" + } + ] + } + }, + "fracSecs": { + "label": "Factional Seconds", + "title": "Number of fractional seconds digits in text streams", + "value": 3, + "set": { + "param": "format.secfrac" + } + }, + "sigDigits": { + "label": "Significant Digits", + "title": "General significant digits for values in text streams", + "value": 5, + "set": { + "param": "format.sigdigit" + } + } + }, + "order": [ + "enabled", + "version", + "serial", + "sigDigits", + "fracSecs" + ] + }, + "csv": { + "label": "CSV stream", + "title": "(Comma) Separated Values for export", + "mimeTypes": [ + "text/csv" + ], + "props": { + "enabled": { + "type": "boolean", + "xorGroup": "format", + "value": false, + "set": { + "value": true, + "param": "format.type", + "pval": "csv" + } + }, + "attributes": { + "value": false, + "type": "boolean", + "label": "Attributes", + "title": "Output data object property rows", + "set": { + "value": true, + "param": "format.csv.props", + "pval": "1" + } + }, + "headers": { + "value": true, + "type": "boolean", + "label": "Headers", + "title": "Output column headers", + "set": { + "value": false, + "param": "format.headers", + "pval": "1" + } + }, + "datasetIds": { + "value": true, + "type": "boolean", + "label": "Dataset IDs", + "title": "Output the dataset ID column", + "pval": "1", + "set": { + "value": false, + "param": "format.csv.dsid", + "pval": "1" + } + }, + "delimiter": { + "value": "Comma", + "type": "enum", + "label": "Value Separator", + "set": { + "param": "format.delim", + "enum": [ + { + "value": "Comma", + "pval": "," + }, + { + "value": "Semicolon", + "pval": ";" + }, + { + "value": "Pipe", + "pval": "|" + }, + { + "value": "Tab", + "pval": "tab" + } + ] + } + }, + "fracSecs": { + "label": "Factional Seconds", + "title": "Number of fractional seconds digits in text streams", + "value": 3, + "set": { + "param": "format.secfrac" + } + }, + "sigDigits": { + "label": "Significant Digits", + "title": "General significant digits for values in text streams", + "value": 5, + "set": { + "param": "format.sigdigit" + } + } + }, + "order": [ + "enabled", + "attributes", + "headers", + "datasetIds", + "delimiter", + "fracSecs", + "sigDigits" + ] + } + } + } +} \ No newline at end of file diff --git a/test/das_fetch/hsk_flex_0.7.json b/test/das_fetch/hsk_flex_0.7.json new file mode 100644 index 0000000..ab3b4f5 --- /dev/null +++ b/test/das_fetch/hsk_flex_0.7.json @@ -0,0 +1,1706 @@ +{ + "label": "HSK", + "name": "TS2_CDPU_HSK", + "title": "TRACERS-2 CDPU: CDPU Internal State Housekeeping Level 1A", + "type": "HttpStreamSrc", + "version": "0.7", + "protocol": { + "method": "GET", + "convention": "HTTP/1.1", + "baseUrls": [ + "https://space.physics.uiowa.edu/das/stream/source/tracers/tracers-2/l1a/cdpu/hsk/flex" + ], + "authorization": { + "required": true, + "methods": [ + "http-basic" + ] + }, + "httpParams": { + "read.time.min": { + "required": true, + "type": "isotime" + }, + "read.time.max": { + "required": true, + "type": "isotime" + }, + "bin.time.max": { + "required": false, + "type": "real", + "units": "s" + }, + "read.data": { + "type": "FlagSet", + "required": false, + "flagSep": ",", + "flags": { + "time": { + "value": "time" + }, + "nmi_errors": { + "value": "nmi_errors" + }, + "irq3_errors": { + "value": "irq3_errors" + }, + "irq4_errors": { + "value": "irq4_errors" + }, + "irq6_errors": { + "value": "irq6_errors" + }, + "irq7_errors": { + "value": "irq7_errors" + }, + "irq8_errors": { + "value": "irq8_errors" + }, + "irqc_errors": { + "value": "irqc_errors" + }, + "nmi_total": { + "value": "nmi_total" + }, + "efi_task_errors": { + "value": "efi_task_errors" + }, + "lvps_task_errors": { + "value": "lvps_task_errors" + }, + "pps_task_errors": { + "value": "pps_task_errors" + }, + "sci_task_errors": { + "value": "sci_task_errors" + }, + "hsk_task_errors": { + "value": "hsk_task_errors" + }, + "memtest_errors": { + "value": "memtest_errors" + }, + "memscrub_errors": { + "value": "memscrub_errors" + }, + "cmd_fifo_errors": { + "value": "cmd_fifo_errors" + }, + "sci_fifo_errors": { + "value": "sci_fifo_errors" + }, + "hsk_fifo_errors": { + "value": "hsk_fifo_errors" + }, + "magic_debug": { + "value": "magic_debug" + }, + "mag_debug": { + "value": "mag_debug" + }, + "msc_debug": { + "value": "msc_debug" + }, + "efi_debug": { + "value": "efi_debug" + }, + "aci_debug": { + "value": "aci_debug" + }, + "ace_debug": { + "value": "ace_debug" + }, + "overc_efi_3p3": { + "value": "overc_efi_3p3" + }, + "overc_efi_p5": { + "value": "overc_efi_p5" + }, + "overc_efi_n5": { + "value": "overc_efi_n5" + }, + "overc_efi_p12": { + "value": "overc_efi_p12" + }, + "overc_efi_n12": { + "value": "overc_efi_n12" + }, + "overc_msc_p12": { + "value": "overc_msc_p12" + }, + "overc_msc_n12": { + "value": "overc_msc_n12" + }, + "overc_mag_p5": { + "value": "overc_mag_p5" + }, + "overc_mag_n5": { + "value": "overc_mag_n5" + }, + "overc_mag_p12": { + "value": "overc_mag_p12" + }, + "overc_mag_n12": { + "value": "overc_mag_n12" + }, + "overc_cdpu_3p3": { + "value": "overc_cdpu_3p3" + }, + "overv_lvps_3p3": { + "value": "overv_lvps_3p3" + }, + "overv_lvps_p5": { + "value": "overv_lvps_p5" + }, + "overv_lvps_n5": { + "value": "overv_lvps_n5" + }, + "overv_lvps_p12": { + "value": "overv_lvps_p12" + }, + "overv_lvps_n12": { + "value": "overv_lvps_n12" + }, + "overt_ace": { + "value": "overt_ace" + }, + "overv_ace_p5": { + "value": "overv_ace_p5" + }, + "underv_ace_p5": { + "value": "underv_ace_p5" + }, + "overv_ace_3p3": { + "value": "overv_ace_3p3" + }, + "underv_ace_3p3": { + "value": "underv_ace_3p3" + }, + "overv_ace_mcp": { + "value": "overv_ace_mcp" + }, + "overc_ace_mcp": { + "value": "overc_ace_mcp" + }, + "overv_ace_stack": { + "value": "overv_ace_stack" + }, + "overc_ace_stack": { + "value": "overc_ace_stack" + }, + "overt_aci": { + "value": "overt_aci" + }, + "overv_aci_p5": { + "value": "overv_aci_p5" + }, + "underv_aci_n5": { + "value": "underv_aci_n5" + }, + "overv_aci_3p3": { + "value": "overv_aci_3p3" + }, + "overv_aci_1p5": { + "value": "overv_aci_1p5" + }, + "overv_aci_mcp": { + "value": "overv_aci_mcp" + }, + "overc_aci_mcp": { + "value": "overc_aci_mcp" + }, + "overv_aci_esa": { + "value": "overv_aci_esa" + }, + "overv_aci_bulk": { + "value": "overv_aci_bulk" + }, + "overc_aci_esa": { + "value": "overc_aci_esa" + }, + "over_aci_count": { + "value": "over_aci_count" + }, + "edac_write_back": { + "value": "edac_write_back" + }, + "edac_mram_enable": { + "value": "edac_mram_enable" + }, + "edac_sram_enable": { + "value": "edac_sram_enable" + }, + "overt_lvps_cnvrt": { + "value": "overt_lvps_cnvrt" + }, + "overt_lvps_chass": { + "value": "overt_lvps_chass" + }, + "overt_lvps_vref": { + "value": "overt_lvps_vref" + }, + "roi_timer": { + "value": "roi_timer" + }, + "edac_mram_errors": { + "value": "edac_mram_errors" + }, + "edac_sram_errors": { + "value": "edac_sram_errors" + }, + "sram_corrected": { + "value": "sram_corrected" + }, + "sram_uncorrected": { + "value": "sram_uncorrected" + } + } + }, + "read.apid": { + "type": "enum", + "required": false, + "enum": [ + "x140" + ] + }, + "format.type": { + "required": false, + "type": "enum", + "enum": [ + "das", + "ccsds" + ] + }, + "format.secfrac": { + "required": false, + "type": "integer", + "range": [ + 0, + 9 + ] + }, + "format.sigdigit": { + "required": false, + "type": "integer", + "range": [ + 2, + 17 + ] + }, + "format.delim": { + "required": false, + "type": "string" + }, + "format.headers": { + "required": false, + "type": "boolean" + }, + "format.csv.props": { + "required": false, + "type": "boolean" + }, + "format.csv.dsid": { + "required": false, + "type": "boolean" + }, + "format.cdf.nocomp": { + "required": false, + "type": "boolean" + }, + "format.serial": { + "label": "Serialization", + "required": false, + "type": "enum", + "enum": [ + "text", + "binary" + ] + }, + "format.version": { + "required": false, + "type": "string" + } + } + }, + "contacts": [ + { + "type": "maintainer", + "name": "Chris Piker", + "email": "chris-piker@uiowa.edu" + }, + { + "type": "scientific", + "name": "Mark Chutter" + } + ], + "interface": { + "coords": { + "time": { + "label": "Time", + "props": { + "min": { + "label": "Minimum", + "value": null, + "title": "Minimum time value to stream", + "set": { + "param": "read.time.min", + "required": true + } + }, + "max": { + "label": "Maximum", + "value": null, + "title": "Maximum time value to stream", + "set": { + "param": "read.time.max", + "required": true + } + }, + "units": { + "value": "UTC" + }, + "res": { + "label": "Resolution", + "value": null, + "units": "s", + "title": "The maximum time bin width for bin-reduced data in seconds", + "set": { + "param": "bin.time.max", + "required": false + } + } + }, + "validRange": [ + "2025-08-04T22:49", + "2026-01-06T14:24" + ] + } + }, + "examples": [ + { + "settings": { + "coords/time/props/min": "2026-01-06T14:12", + "coords/time/props/max": "2026-01-06T14:24" + }, + "label": "Most Recent 10 minutes (das3)" + }, + { + "settings": { + "coords/time/props/min": "2026-01-05T14:24", + "coords/time/props/max": "2026-01-06T14:24", + "formats/cdf/props/enabled": true + }, + "label": "Most Recent Day (CDF)" + } + ], + "formats": { + "cdf": { + "label": "CDF file", + "title": "NASA Common Data Format", + "mimeTypes": [ + "application/x-cdf" + ], + "props": { + "enabled": { + "type": "boolean", + "xorGroup": "format", + "value": false, + "set": { + "value": true, + "param": "format.type", + "pval": "cdf" + } + }, + "nocompress": { + "value": false, + "type": "boolean", + "label": "No-Compression", + "title": "Do not compress variables, needed for PDS compliant output.", + "set": { + "value": true, + "param": "format.cdf.nocomp", + "pval": "1" + } + } + } + }, + "das": { + "label": "das stream", + "title": "Streaming format for plots", + "mimeTypes": [ + "application/vnd.das2.das2stream", + "application/vnd.das.stream", + "text/vnd.das2.das2stream" + ], + "props": { + "enabled": { + "type": "boolean", + "xorGroup": "format", + "value": true, + "set": { + "value": true, + "param": "format.type", + "pval": "das" + } + }, + "version": { + "label": "Stream Version", + "value": "das3", + "type": "enum", + "set": { + "param": "format.version", + "enum": [ + { + "value": "das3", + "pval": "3" + }, + { + "value": "das2", + "pval": "2" + } + ] + } + }, + "serial": { + "label": "Serialization", + "value": "binary", + "set": { + "param": "format.serial", + "enum": [ + { + "value": "text" + }, + { + "value": "binary" + } + ] + } + }, + "fracSecs": { + "label": "Factional Seconds", + "title": "Number of fractional seconds digits in text streams", + "value": 3, + "set": { + "param": "format.secfrac" + } + }, + "sigDigits": { + "label": "Significant Digits", + "title": "General significant digits for values in text streams", + "value": 5, + "set": { + "param": "format.sigdigit" + } + } + }, + "order": [ + "enabled", + "version", + "serial", + "sigDigits", + "fracSecs" + ] + }, + "csv": { + "label": "CSV stream", + "title": "(Comma) Separated Values for export", + "mimeTypes": [ + "text/csv" + ], + "props": { + "enabled": { + "type": "boolean", + "xorGroup": "format", + "value": false, + "set": { + "value": true, + "param": "format.type", + "pval": "csv" + } + }, + "attributes": { + "value": false, + "type": "boolean", + "label": "Attributes", + "title": "Output data object property rows", + "set": { + "value": true, + "param": "format.csv.props", + "pval": "1" + } + }, + "headers": { + "value": true, + "type": "boolean", + "label": "Headers", + "title": "Output column headers", + "set": { + "value": false, + "param": "format.headers", + "pval": "1" + } + }, + "datasetIds": { + "value": true, + "type": "boolean", + "label": "Dataset IDs", + "title": "Output the dataset ID column", + "pval": "1", + "set": { + "value": false, + "param": "format.csv.dsid", + "pval": "1" + } + }, + "delimiter": { + "value": "Comma", + "type": "enum", + "label": "Value Separator", + "set": { + "param": "format.delim", + "enum": [ + { + "value": "Comma", + "pval": "," + }, + { + "value": "Semicolon", + "pval": ";" + }, + { + "value": "Pipe", + "pval": "|" + }, + { + "value": "Tab", + "pval": "tab" + } + ] + } + }, + "fracSecs": { + "label": "Factional Seconds", + "title": "Number of fractional seconds digits in text streams", + "value": 3, + "set": { + "param": "format.secfrac" + } + }, + "sigDigits": { + "label": "Significant Digits", + "title": "General significant digits for values in text streams", + "value": 5, + "set": { + "param": "format.sigdigit" + } + } + }, + "order": [ + "enabled", + "attributes", + "headers", + "datasetIds", + "delimiter", + "fracSecs", + "sigDigits" + ] + } + }, + "data": { + "time": { + "label": "time", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "time" + } + } + } + }, + "nmi_errors": { + "label": "NMI Errs", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "nmi_errors" + } + } + }, + "title": "NMI Error Counter" + }, + "irq3_errors": { + "label": "Cnt IRQ3 Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "irq3_errors" + } + } + }, + "title": "IRQ-3 error counter" + }, + "irq4_errors": { + "label": "Cnt IRQ4 Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "irq4_errors" + } + } + }, + "title": "IRQ-4 error counter" + }, + "irq6_errors": { + "label": "Cnt IRQ6 Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "irq6_errors" + } + } + }, + "title": "IRQ-6 error counter" + }, + "irq7_errors": { + "label": "Cnt IRQ7 Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "irq7_errors" + } + } + }, + "title": "IRQ-7 error counter" + }, + "irq8_errors": { + "label": "Cnt IRQ8 Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "irq8_errors" + } + } + }, + "title": "IRQ-8 error counter" + }, + "irqc_errors": { + "label": "Cnt IRQC Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "irqc_errors" + } + } + }, + "title": "IRQ-C error counter" + }, + "nmi_total": { + "label": "Count NMI", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "nmi_total" + } + } + }, + "title": "count_nmi warning counter" + }, + "efi_task_errors": { + "label": "EFI Task Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "efi_task_errors" + } + } + }, + "title": "efi_task_error error counter" + }, + "lvps_task_errors": { + "label": "LVPS Task Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "lvps_task_errors" + } + } + }, + "title": "lvps_task_error error counter" + }, + "pps_task_errors": { + "label": "IRQD Task Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "pps_task_errors" + } + } + }, + "title": "irqd_task_error error counter" + }, + "sci_task_errors": { + "label": "Sci Task Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "sci_task_errors" + } + } + }, + "title": "sci_task_error error counter" + }, + "hsk_task_errors": { + "label": "HK Task Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "hsk_task_errors" + } + } + }, + "title": "hk_task_error error counter" + }, + "memtest_errors": { + "label": "Test Task Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "memtest_errors" + } + } + }, + "title": "test_task_error error counter" + }, + "memscrub_errors": { + "label": "Scrb Task Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "memscrub_errors" + } + } + }, + "title": "scrub_task_error error counter" + }, + "cmd_fifo_errors": { + "label": "TC FIFO Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "cmd_fifo_errors" + } + } + }, + "title": "tc_fifo_error error counter" + }, + "sci_fifo_errors": { + "label": "SC FIFO Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "sci_fifo_errors" + } + } + }, + "title": "sc_fifo_error error counter" + }, + "hsk_fifo_errors": { + "label": "HK FIFO Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "hsk_fifo_errors" + } + } + }, + "title": "hk_fifo_error error counter" + }, + "magic_debug": { + "label": "MAGIC Debug", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "magic_debug" + } + } + }, + "title": "magic_tm debug flag" + }, + "mag_debug": { + "label": "MAG Debug", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "mag_debug" + } + } + }, + "title": "mag_tm debug flag" + }, + "msc_debug": { + "label": "MSC Debug", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "msc_debug" + } + } + }, + "title": "msc_tm debug flag" + }, + "efi_debug": { + "label": "EFI Debug", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "efi_debug" + } + } + }, + "title": "efi_tm debug flag" + }, + "aci_debug": { + "label": "ACI Debug", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "aci_debug" + } + } + }, + "title": "aci_tm debug flag" + }, + "ace_debug": { + "label": "ACE Debug", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "ace_debug" + } + } + }, + "title": "ace_tm debug flag" + }, + "overc_efi_3p3": { + "label": "EFI 3.3V OC", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overc_efi_3p3" + } + } + }, + "title": "Over-current flag for EFI 3.3V" + }, + "overc_efi_p5": { + "label": "EFI +5V OC", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overc_efi_p5" + } + } + }, + "title": "Over-current flag for EFI +5V" + }, + "overc_efi_n5": { + "label": "EFI -5V OC", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overc_efi_n5" + } + } + }, + "title": "Over-current flag for EFI -5V" + }, + "overc_efi_p12": { + "label": "EFI +12V OC", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overc_efi_p12" + } + } + }, + "title": "Over-current flag for EFI +12V" + }, + "overc_efi_n12": { + "label": "EFI -12V OC", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overc_efi_n12" + } + } + }, + "title": "Over-current flag for EFI -12V" + }, + "overc_msc_p12": { + "label": "MSC +12V OC", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overc_msc_p12" + } + } + }, + "title": "Over-current flag for MSC +12V" + }, + "overc_msc_n12": { + "label": "MSC -12V OC", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overc_msc_n12" + } + } + }, + "title": "Over-current flag for MSC -12V" + }, + "overc_mag_p5": { + "label": "MAG +5V OC", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overc_mag_p5" + } + } + }, + "title": "Over-current flag for MAG +5V" + }, + "overc_mag_n5": { + "label": "MAG -5V OC", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overc_mag_n5" + } + } + }, + "title": "Over-current flag for MAG -5V" + }, + "overc_mag_p12": { + "label": "MAG +12V OC", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overc_mag_p12" + } + } + }, + "title": "Over-current flag for MAG +12V" + }, + "overc_mag_n12": { + "label": "MAG -12V OC", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overc_mag_n12" + } + } + }, + "title": "Over-current flag for MAG -12V" + }, + "overc_cdpu_3p3": { + "label": "CDPU OC", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overc_cdpu_3p3" + } + } + }, + "title": "Over-current flag for CDPU" + }, + "overv_lvps_3p3": { + "label": "LVPS 3.3V OV", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overv_lvps_3p3" + } + } + }, + "title": "Over-voltage flag for LVPS 3.3V" + }, + "overv_lvps_p5": { + "label": "LVPS +5V OV", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overv_lvps_p5" + } + } + }, + "title": "Over-voltage flag for LVPS +5V" + }, + "overv_lvps_n5": { + "label": "LVPS -5V OV", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overv_lvps_n5" + } + } + }, + "title": "Over-voltage flag for LVPS -5V" + }, + "overv_lvps_p12": { + "label": "LVPS +12V OV", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overv_lvps_p12" + } + } + }, + "title": "Over-voltage flag for LVPS +12V" + }, + "overv_lvps_n12": { + "label": "LVPS -12V OV", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overv_lvps_n12" + } + } + }, + "title": "Over-voltage flag for LVPS -12V" + }, + "overt_ace": { + "label": "ACE OT", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overt_ace" + } + } + }, + "title": "Over-temp flag for ACE" + }, + "overv_ace_p5": { + "label": "ACE +5V OV", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overv_ace_p5" + } + } + }, + "title": "Over-voltage flag for ACE +5V" + }, + "underv_ace_p5": { + "label": "ACE +5V UV", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "underv_ace_p5" + } + } + }, + "title": "Under-voltage flag for ACE +5V" + }, + "overv_ace_3p3": { + "label": "ACE 3.3V OV", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overv_ace_3p3" + } + } + }, + "title": "Over-voltage flag for ACE 3.3V" + }, + "underv_ace_3p3": { + "label": "ACE 3.3V UV", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "underv_ace_3p3" + } + } + }, + "title": "Under-voltage flag for ACE 3.3V" + }, + "overv_ace_mcp": { + "label": "ACE MCP OV", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overv_ace_mcp" + } + } + }, + "title": "Over-voltage flag for ACE MCP" + }, + "overc_ace_mcp": { + "label": "ACE MCP OC", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overc_ace_mcp" + } + } + }, + "title": "Over-current flag for ACE MCP" + }, + "overv_ace_stack": { + "label": "ACE Stack OV", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overv_ace_stack" + } + } + }, + "title": "Over-voltage flag for ACE Stack" + }, + "overc_ace_stack": { + "label": "ACE Stack OC", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overc_ace_stack" + } + } + }, + "title": "Over-current flag for ACE Stack" + }, + "overt_aci": { + "label": "ACI OT", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overt_aci" + } + } + }, + "title": "Over-temp flag for ACI" + }, + "overv_aci_p5": { + "label": "ACI +5V OV", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overv_aci_p5" + } + } + }, + "title": "Over-voltage flag for ACI +5V" + }, + "underv_aci_n5": { + "label": "ACI -5V UV", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "underv_aci_n5" + } + } + }, + "title": "Under-voltage flag for ACI -5V" + }, + "overv_aci_3p3": { + "label": "ACI 3.3V OV", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overv_aci_3p3" + } + } + }, + "title": "Over-voltage flag for ACI 3.3V" + }, + "overv_aci_1p5": { + "label": "ACI 1.5V OV", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overv_aci_1p5" + } + } + }, + "title": "Over-voltage flag for ACI 1.5V" + }, + "overv_aci_mcp": { + "label": "ACI MCP OV", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overv_aci_mcp" + } + } + }, + "title": "Over-voltage flag for ACI MCP" + }, + "overc_aci_mcp": { + "label": "ACI MCP OC", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overc_aci_mcp" + } + } + }, + "title": "Over-current flag for ACI MCP" + }, + "overv_aci_esa": { + "label": "ACI ESA OV", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overv_aci_esa" + } + } + }, + "title": "Over-voltage flag for ACI ESA" + }, + "overv_aci_bulk": { + "label": "ACI Bulk OV", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overv_aci_bulk" + } + } + }, + "title": "Over-voltage flag for ACI Bulk" + }, + "overc_aci_esa": { + "label": "ACI ESA OC", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overc_aci_esa" + } + } + }, + "title": "Over-current flag for ACI ESA" + }, + "over_aci_count": { + "label": "ACI OverCnt", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "over_aci_count" + } + } + }, + "title": "Over-count flag for ACI" + }, + "edac_write_back": { + "label": "EDAC WrtBk", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "edac_write_back" + } + } + }, + "title": "EDAC write back active" + }, + "edac_mram_enable": { + "label": "EDAC MRAM", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "edac_mram_enable" + } + } + }, + "title": "MRAM EDAC checking active" + }, + "edac_sram_enable": { + "label": "EDAC SRAM", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "edac_sram_enable" + } + } + }, + "title": "SRAM EDAC checking active" + }, + "overt_lvps_cnvrt": { + "label": "LVPS Cnv OT", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overt_lvps_cnvrt" + } + } + }, + "title": "Over-temp flag for LVPS converter" + }, + "overt_lvps_chass": { + "label": "LVPS Chs OT", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overt_lvps_chass" + } + } + }, + "title": "Over-temp flag for LVPS chassis" + }, + "overt_lvps_vref": { + "label": "LVPS Vref OT", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overt_lvps_vref" + } + } + }, + "title": "Over-temp flag for LVPS vref" + }, + "roi_timer": { + "label": "ROI CntDn Tmr", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "roi_timer" + } + } + }, + "title": "ROI countdown timer" + }, + "edac_mram_errors": { + "label": "EDAC MRAM Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "edac_mram_errors" + } + } + }, + "title": "Counter of EDAC MRAM errors" + }, + "edac_sram_errors": { + "label": "EDAC SRAM Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "edac_sram_errors" + } + } + }, + "title": "Counter of EDAC SRAM errors" + }, + "sram_corrected": { + "label": "SBit Cor Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "sram_corrected" + } + } + }, + "title": "Count of single-bit correctable errors" + }, + "sram_uncorrected": { + "label": "EDAC SRAM Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "sram_uncorrected" + } + } + }, + "title": "Counter of EDAC SRAM errors" + } + } + } +} \ No newline at end of file diff --git a/test/das_fetch/juno_survey_0.6.json b/test/das_fetch/juno_survey_0.6.json new file mode 100644 index 0000000..be552b3 --- /dev/null +++ b/test/das_fetch/juno_survey_0.6.json @@ -0,0 +1,348 @@ +{ + "format": { + "default": { + "mime": "application/vnd.das2.das2stream", + "name": "Das2 Stream" + } + }, + "interface": { + "coordinates": { + "frequency": { + "name": "Frequency", + "title": "Y", + "units": { + "value": "Hz" + } + }, + "time": { + "maximum": { + "set": { + "param": "end_time", + "range": [ + "2010-09-01T00:00:00", + "now" + ], + "required": true + }, + "value": "2018-10-04T11:00" + }, + "minimum": { + "set": { + "param": "start_time", + "range": [ + "2010-09-01T00:00:00", + "now" + ], + "required": true + }, + "value": "2018-10-02T11:00" + }, + "name": "Time", + "resolution": { + "set": { + "param": "resolution", + "required": false + }, + "units": "s", + "value": 86.4 + }, + "units": { + "value": "UTC" + } + } + }, + "data": { + "specdens": { + "name": "Specdens", + "title": "Z", + "units": { + "value": "V**2 m**-2 Hz**-1" + } + } + }, + "options": { + "ascii": { + "set": { + "flag": "00", + "param": "params", + "value": true + }, + "title": "If specified, output is in UTF-8 format", + "type": "boolean", + "value": false + }, + "b": { + "set": { + "flag": "03", + "param": "params", + "value": true + }, + "title": "Only output magnetic data from the low-band of the Low Frequency Receiver", + "type": "boolean", + "value": false + }, + "das2_text": { + "name": "das2 text", + "set": { + "param": "ascii", + "pval": "true", + "value": true + }, + "title": "Convert stream to das2 text (utf-8) format", + "value": false + }, + "hfrh": { + "set": { + "flag": "07", + "param": "params", + "value": true + }, + "title": "Only output data from the frequency down-mixed sections of the High Frequency Receivers", + "type": "boolean", + "value": false + }, + "hfrl": { + "set": { + "flag": "06", + "param": "params", + "value": true + }, + "title": "Only output data from the baseband of the High Frequency Receivers", + "type": "boolean", + "value": false + }, + "keep-spikes": { + "set": { + "flag": "01", + "param": "params", + "value": true + }, + "title": "If specified single LFR packets with a pre-amp toggle are retained in the output", + "type": "boolean", + "value": false + }, + "lfrh": { + "set": { + "flag": "05", + "param": "params", + "value": true + }, + "title": "Only output data from the high-band of the Low Frequency Receiver", + "type": "boolean", + "value": false + }, + "lfrl": { + "set": { + "flag": "04", + "param": "params", + "value": true + }, + "title": "Only output electric from the low-band of the Low Frequency Receiver", + "type": "boolean", + "value": false + }, + "xmas": { + "set": { + "flag": "02", + "param": "params", + "value": true + }, + "title": "If specified only output the Xmas flag", + "type": "boolean", + "value": false + } + } + }, + "name": "Das2/2.2 Source", + "protocol": { + "authentication": { + "realm": "Juno Magnetospheric Working Group", + "required": true + }, + "base_urls": [ + "http://127.0.0.1:8765/data/juno_survey.d2s?server=dataset&dataset=Juno/WAV/Survey" + ], + "convention": "das2/2.2", + "examples": { + "example_00": { + "http_params": { + "end_time": "2018-10-04T11:00", + "resolution": 86.4, + "start_time": "2018-10-02T11:00" + }, + "name": "Example 00", + "title": "Perijove 16", + "url": "https://jupiter.physics.uiowa.edu/das/server?server=dataset&dataset=Juno/WAV/Survey&resolution=86.4&end_time=2018-10-04T11%3A00&start_time=2018-10-02T11%3A00" + }, + "example_01": { + "http_params": { + "end_time": "2018-08-12T15:00", + "resolution": 86.4, + "start_time": "2018-08-10T15:00" + }, + "name": "Example 01", + "title": "Perijove 15", + "url": "https://jupiter.physics.uiowa.edu/das/server?server=dataset&dataset=Juno/WAV/Survey&resolution=86.4&end_time=2018-08-12T15%3A00&start_time=2018-08-10T15%3A00" + }, + "example_02": { + "http_params": { + "end_time": "2018-06-20T18:00", + "resolution": 86.4, + "start_time": "2018-06-18T18:00" + }, + "name": "Example 02", + "title": "Perijove 14", + "url": "https://jupiter.physics.uiowa.edu/das/server?server=dataset&dataset=Juno/WAV/Survey&resolution=86.4&end_time=2018-06-20T18%3A00&start_time=2018-06-18T18%3A00" + }, + "example_03": { + "http_params": { + "end_time": "2018-04-28T20:00", + "resolution": 86.4, + "start_time": "2018-04-26T20:00" + }, + "name": "Example 03", + "title": "Perijove 13", + "url": "https://jupiter.physics.uiowa.edu/das/server?server=dataset&dataset=Juno/WAV/Survey&resolution=86.4&end_time=2018-04-28T20%3A00&start_time=2018-04-26T20%3A00" + }, + "example_04": { + "http_params": { + "end_time": "2018-03-07T00:00", + "resolution": 86.4, + "start_time": "2018-03-05T00:00" + }, + "name": "Example 04", + "title": "Perijove 12", + "url": "https://jupiter.physics.uiowa.edu/das/server?server=dataset&dataset=Juno/WAV/Survey&resolution=86.4&end_time=2018-03-07T00%3A00&start_time=2018-03-05T00%3A00" + }, + "example_05": { + "http_params": { + "end_time": "2018-01-13T04:00", + "resolution": 86.4, + "start_time": "2018-01-11T04:00" + }, + "name": "Example 05", + "title": "Perijove 11", + "url": "https://jupiter.physics.uiowa.edu/das/server?server=dataset&dataset=Juno/WAV/Survey&resolution=86.4&end_time=2018-01-13T04%3A00&start_time=2018-01-11T04%3A00" + }, + "example_15": { + "http_params": { + "end_time": "2016-177T00:00", + "resolution": 43.2, + "start_time": "2016-176T00:00" + }, + "name": "Example 15", + "title": "Jupiter Bow Shock", + "url": "https://jupiter.physics.uiowa.edu/das/server?server=dataset&dataset=Juno/WAV/Survey&resolution=43.2&end_time=2016-177T00%3A00&start_time=2016-176T00%3A00" + }, + "example_16": { + "http_params": { + "end_time": "2016-177T00:00", + "params": "B", + "resolution": 43.2, + "start_time": "2016-176T00:00" + }, + "name": "Example 16", + "title": "Jupiter Bow Shock, Magnetic", + "url": "https://jupiter.physics.uiowa.edu/das/server?server=dataset&dataset=Juno/WAV/Survey&resolution=43.2&end_time=2016-177T00%3A00¶ms=B&start_time=2016-176T00%3A00" + }, + "example_98": { + "http_params": { + "end_time": "2013-10-10", + "resolution": 43.2, + "start_time": "2013-10-09" + }, + "name": "Example 98", + "title": "Earth Flyby", + "url": "https://jupiter.physics.uiowa.edu/das/server?server=dataset&dataset=Juno/WAV/Survey&resolution=43.2&end_time=2013-10-10&start_time=2013-10-09" + } + }, + "http_params": { + "ascii": { + "name": "UTF-8", + "required": false, + "title": "Insure stream output is readable as UTF-8 text", + "type": "boolean" + }, + "end_time": { + "name": "Max Time", + "required": true, + "title": "Maximum Time Value to stream", + "type": "isotime" + }, + "params": { + "flag_sep": " ", + "flags": { + "00": { + "name": "ASCII", + "title": "If specified, output is in UTF-8 format", + "value": "ASCII" + }, + "01": { + "name": "KEEP-SPIKES", + "title": "If specified single LFR packets with a pre-amp toggle are retained in the output", + "value": "KEEP-SPIKES" + }, + "02": { + "name": "XMAS", + "title": "If specified only output the Xmas flag", + "value": "XMAS" + }, + "03": { + "name": "B", + "title": "Only output magnetic data from the low-band of the Low Frequency Receiver", + "value": "B" + }, + "04": { + "name": "LFRL", + "title": "Only output electric from the low-band of the Low Frequency Receiver", + "value": "LFRL" + }, + "05": { + "name": "LFRH", + "title": "Only output data from the high-band of the Low Frequency Receiver", + "value": "LFRH" + }, + "06": { + "name": "HFRL", + "title": "Only output data from the baseband of the High Frequency Receivers", + "value": "HFRL" + }, + "07": { + "name": "HFRH", + "title": "Only output data from the frequency down-mixed sections of the High Frequency Receivers", + "value": "HFRH" + } + }, + "required": false, + "title": "Optional reader arguments", + "type": "flag_set" + }, + "resolution": { + "description": "The server will return data at or better than the given resolution if possible. Leave un-specified to get data at intrinsic resolution without server side averages", + "name": "Resolution", + "required": false, + "title": "Maximum resolution between output time points", + "type": "real", + "units": "s" + }, + "start_time": { + "name": "Min Time", + "required": true, + "title": "Minimum time value to stream", + "type": "isotime" + } + } + }, + "tech_contacts": [ + { + "email": "chris-piker@uiowa.edu", + "name": "Chris Piker" + } + ], + "title": "Survey Mode Electric or Magnetic Spectral Densities", + "type": "HttpStreamSrc", + "uris": [ + "tag:physics.uiowa.edu,2018:das.juno.wav.survey.das2" + ], + "version": "0.6" +} \ No newline at end of file diff --git a/test/das_fetch_test.sh b/test/das_fetch_test.sh new file mode 100755 index 0000000..c9c13bc --- /dev/null +++ b/test/das_fetch_test.sh @@ -0,0 +1,217 @@ +#!/usr/bin/env bash + +# Regression test for das_fetch's interface-to-protocol translation. Runs +# entirely offline: the fixtures in test/das_fetch/ are real catalog nodes +# saved from the uiowa servers with their base URLs pointed at localhost, +# and the 'url' command resolves a query without contacting anything. +# +# Three node generations are covered: +# geomag_flex_0.7.json dasflex HttpStreamSrc 0.7, required interval param, +# FlagSet data toggle, csv/cdf/das format groups +# hsk_flex_0.7.json 0.7 with 73 data variables and a das2/das3 version enum +# juno_survey_0.6.json 2018 era HttpStreamSrc 0.6, flag_set with space +# separator, base URL that already has a query string +# +# Each case is: arguments, then the exact URL expected on stdout. Error cases +# check the exit code instead. +# +# Drafted by Claude Fable 5.1, filed by Chris Piker. + +BD=$1 +FETCH="$BD/das_fetch -C none" # no node cache, the fixtures are files anyway +FX=test/das_fetch + +nFail=0 +nPass=0 + +# expect_url ARGS... -- EXPECTED +expect_url() { + local args=() + while [ "$1" != "--" ]; do args+=("$1"); shift; done + shift + local expected="$1" + local actual + actual=$($FETCH url "${args[@]}" 2>/dev/null) + if [ "$actual" == "$expected" ]; then + nPass=$((nPass + 1)) + else + echo " FAIL: das_fetch url ${args[*]}" + echo " expected: $expected" + echo " actual: $actual" + nFail=$((nFail + 1)) + fi +} + +# expect_rc CODE ARGS... +expect_rc() { + local want=$1; shift + $FETCH "$@" > /dev/null 2>&1 + local got=$? + if [ $got -eq $want ]; then + nPass=$((nPass + 1)) + else + echo " FAIL: das_fetch $* : expected exit $want, got $got" + nFail=$((nFail + 1)) + fi +} + +G=$FX/geomag_flex_0.7.json +H=$FX/hsk_flex_0.7.json +J=$FX/juno_survey_0.6.json +GB="http://127.0.0.1:8765/data/geomag.d2s" +HB="https://space.physics.uiowa.edu/das/stream/source/tracers/tracers-2/l1a/cdpu/hsk/flex" +JB="http://127.0.0.1:8765/data/juno_survey.d2s?server=dataset&dataset=Juno/WAV/Survey" + +# --- 0.7 geomag: ranges, shortcuts, aliases, flags, examples, formats -------- + +expect_url $G time=2026-04-01,2026-04-01T01:30,60 lshell -- \ + "$GB?read.time.min=2026-04-01&read.time.max=2026-04-01T01%3A30&read.time.inter=60&read.data=lshell" + +# minimum/maximum spellings work on a 0.7 node, section prefix optional +expect_url $G coord.time.minimum=2026-02-01 time.maximum=2026-02-02 inter=60 -- \ + "$GB?read.time.min=2026-02-01&read.time.max=2026-02-02&read.time.inter=60" + +# example 0, then an explicit key overrides one of its settings +expect_url $G 0 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" +expect_url $G 0 time.inter=30 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=30" + +# a boolean set to its default is not sent +expect_url $G 0 lshell=no -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" + +# format group by name, enum pval substitution (Tab -> tab), boolean with pval +expect_url $G 0 csv delimiter=Tab csv.headers=false serial=text -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60&format.serial=text&format.type=csv&format.headers=1&format.delim=tab" + +# example by bare number, by examples.N, and as the default when nothing is asked +expect_url $G 0 time.inter=30 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=30" +expect_url $G examples.0 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" +expect_url $G -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" + +# -r may name the root node itself, PATH is then optional +expect_url -r $G 0 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" + +# a seed, URI=URL: the node at URL is named URI, relative paths hang off it +SU="tag:das2.org,2012:site:/uiowa/tracers/tracers-2/spice/geomagcoords/flex" +expect_url -r $SU=$G 0 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" +expect_url -r $SU=$G $SU 0 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" +if $FETCH node -r $SU=$G 2>/dev/null | grep -q "^URI: $SU\$"; then + nPass=$((nPass + 1)) +else + echo " FAIL: das_fetch node -r $SU=$G did not report the seed URI"; nFail=$((nFail + 1)) +fi + +# -g grafts the same seed over the federated catalog. Offline only the +# seed side can be exercised; -g and -r URL together is a usage error. +expect_url -g $SU=$G 0 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" +export DASFETCH_GRAFT=$SU=$G +expect_url 0 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" +unset DASFETCH_GRAFT +expect_rc 2 url -g $SU=$G -r $H 0 # two roots +expect_rc 2 url -g site:/uiowa 0 # graft needs URI=URL +expect_rc 2 url -r site:/uiowa=nourl 0 # '=' but no URL after it +expect_rc 3 url -r $SU=$G site:/uiowa/juno # -r seed is the whole catalog + +# DASFETCH_ROOT stands in for -r, and -r still wins over it +export DASFETCH_ROOT=$G +expect_url 0 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" +expect_url -r $H 0 -- \ + "$HB?read.time.min=2026-01-06T14%3A12&read.time.max=2026-01-06T14%3A24" +unset DASFETCH_ROOT + +# dotted member paths, section names may be plural or singular +expect_url $G 0 data.lshell coord.time.inter=30 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=30&read.data=lshell" + +# raw protocol parameters override translated ones and may add new ones +expect_url $G 0 -p read.time.inter=5 -p extra=1 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=5&extra=1" + +# --- 0.7 hsk: version enum, FlagSet accumulation with comma, cdf example ---- + +# the node default (das3) is not sent, only choices are +expect_url $H 0 -- \ + "$HB?read.time.min=2026-01-06T14%3A12&read.time.max=2026-01-06T14%3A24" +expect_url $H 0 version=das2 nmi_errors irq3_errors data.hsk_task_errors -- \ + "$HB?read.time.min=2026-01-06T14%3A12&read.time.max=2026-01-06T14%3A24&read.data=nmi_errors%2Cirq3_errors%2Chsk_task_errors&format.version=2" +expect_url $H 1 -- \ + "$HB?read.time.min=2026-01-05T14%3A24&read.time.max=2026-01-06T14%3A24&format.type=cdf" +expect_url $H 0 formats.csv das.version=das2 time.res=10 -- \ + "$HB?read.time.min=2026-01-06T14%3A12&read.time.max=2026-01-06T14%3A24&bin.time.max=10&format.version=2&format.type=csv" + +# --- 0.6 juno: '&' joiner, space separated flags, protocol level examples --- + +expect_url $J time=2021-02-21,2021-02-22 b lfrl das2_text -- \ + "$JB&end_time=2021-02-22&start_time=2021-02-21¶ms=B%20LFRL&ascii=true" +expect_url $J time=2021-02-21,2021-02-22,0 ascii option.xmas=true -- \ + "$JB&end_time=2021-02-22&start_time=2021-02-21&resolution=0¶ms=ASCII%20XMAS" + +# a 0.6 example is raw parameters; user keys still win, required defaults +# must not clobber the example's values +expect_url $J 7 resolution=10 -- \ + "$JB&end_time=2016-177T00%3A00¶ms=B&resolution=10&start_time=2016-176T00%3A00" + +# --- error paths, by exit code ---------------------------------------------- + +expect_rc 4 url $G time=2026-04-01,2026-04-01T01:30 # required inter missing +expect_rc 4 url $G 0 foo=1 # no such key +expect_rc 4 url $G 0 fracSecs=4 # ambiguous (das, csv) +expect_rc 4 url $G 0 formats.xml # no such format +expect_rc 4 url $G 0 lshell=maybe # not a boolean +expect_rc 4 url $G 0 delimiter=Colon # not in enum +expect_rc 4 url $G 0 version=das3 # geomag offers only das2 +expect_rc 4 url $G 9 # no such example +expect_rc 4 url $G example=0 # sections are not keys +expect_rc 4 url $G examples.one # example must be a number +expect_rc 4 url $G 0 data.nope # no such data variable +expect_rc 2 node -s bogus $G # bad section list +expect_rc 2 url # missing path +expect_rc 2 bogus $G # unknown command +expect_rc 3 url $FX/does_not_exist.json 0 # unreadable node + +# formats are hidden by default, -s or -v brings them back +if $FETCH node $G 2>/dev/null | grep -q "delimiter"; then + echo " FAIL: das_fetch node $G showed format keys without -s"; nFail=$((nFail + 1)) +else + nPass=$((nPass + 1)) +fi +if $FETCH node -s formats $G 2>/dev/null | grep -q "delimiter"; then + nPass=$((nPass + 1)) +else + echo " FAIL: das_fetch node -s formats $G hid the format keys"; nFail=$((nFail + 1)) +fi + +# node and json commands run on every fixture and print the key table header +for f in $G $H $J; do + if $FETCH node $f 2>/dev/null | grep -q "^ Key "; then + nPass=$((nPass + 1)) + else + echo " FAIL: das_fetch node $f printed no key table" + nFail=$((nFail + 1)) + fi + if $FETCH json "$f#protocol" 2>/dev/null | grep -q '"method"\|"convention"'; then + nPass=$((nPass + 1)) + else + echo " FAIL: das_fetch json $f#protocol printed no protocol block" + nFail=$((nFail + 1)) + fi +done + +if [ $nFail -eq 0 ]; then + echo " Result: PASSED ($nPass checks)" + exit 0 +else + echo " Result: FAILED ($nFail of $((nPass + nFail)) checks)" + exit 1 +fi diff --git a/utilities/das_fetch.c b/utilities/das_fetch.c new file mode 100644 index 0000000..bd2fae2 --- /dev/null +++ b/utilities/das_fetch.c @@ -0,0 +1,2552 @@ +/* Copyright (C) 2026 Chris Piker + * + * Author: Claude Fable 5.1 (Anthropic), from an idea by Chris Piker + * + * This file is part of das2C, the Core Das C Library. + * + * das2C is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License version 2.1 as published + * by the Free Software Foundation. + * + * das2C is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for + * more details. + * + * You should have received a copy of the GNU Lesser General Public License + * version 2.1 along with das2C; if not, see . + */ + +/* das_fetch: walk the das federated catalog, describe data sources, and + * stream their output to stdout, all by catalog path. Nobody types a + * server URL. + * + * The program has two halves. The small half is catalog navigation, which + * is almost entirely the library's job (das3/node.h). The large half is + * turning a data source's "interface" section into command line keys, and + * turning the user's KEY=VALUE arguments back into HTTP GET parameters via + * the source's "protocol" section. That translation follows the same rules + * das2py uses in streamsrc.HttpStreamSrc._translate(), so the two clients + * agree on what a source definition means. + * + * Two generations of HttpStreamSrc node are in the wild (0.6 from 2018, + * 0.7 from dasflex) and they spell things differently. Layout detection + * here is structural (does a group have a "props" sub-dictionary) rather + * than by version string, and the 0.6 spellings are handled inline where + * they differ. Once the 0.6 nodes are retired those branches can go. + * + * The body of a data response is never inspected. It may be a das2 or das3 + * stream, a CDF file, CSV text, or CCSDS packets. Bytes are bytes. + */ + +#define _POSIX_C_SOURCE 200112L + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#ifdef _WIN32 +#include +#include +#define isatty _isatty +#define fileno _fileno +#else +#include +#include +#endif + +#define PROG "das_fetch" +#define PERR (DASERR_MAX + 1) + +#define DEF_ROOT "tag:das2.org,2012:site:/" +#define DAS_DIR ".das" /* under the home directory, holds cache/ and servers */ +#define CRED_FILE "servers" /* das2C leads here, das2py follows; ~/.dasauth is not read */ + +#ifndef _WIN32 +#define HOME_VAR "$HOME" /* for help text, das_userhome() does the lookup */ +#else +#define HOME_VAR "%USERPROFILE%" +#endif + +/* Exit codes. Documented in the help text so shell scripts can branch on + * them instead of parsing stderr. */ +#define RET_OK 0 +#define RET_USAGE 2 +#define RET_NODE 3 +#define RET_QUERY 4 +#define RET_SERVER 5 +#define RET_AUTH 6 +#define RET_WRITE 7 + +#define MAX_SETTABLES 512 +#define MAX_PARAMS 96 +#define MAX_RAW 32 +/* Identifiers in a node (property, variable and group names) follow the das3 + * rule enforced by das_assert_valid_id(): at most 63 characters. A full key + * is "section.group.prop", so size it from that rule, not from a guess. */ +#define NAME_SZ DAS_MAX_ID_BUFSZ +#define KEY_SZ (8 + 2*DAS_MAX_ID_BUFSZ) +#define VAL_SZ 512 +#define URI_SZ 1024 +#define URL_SZ 4096 +#define ERR_SZ 1024 +#define CHUNK_SZ (64*1024) + +#define DEF_CACHE_AGE 3600 /* seconds a cached catalog node is trusted */ +#define ROOT_ENV "DASFETCH_ROOT" +#define GRAFT_ENV "DASFETCH_GRAFT" + +/* ************************************************************************* */ +/* Help */ + +static void prnHelpMain(void) +{ + fputs( +"NAME\n" +" " PROG " - fetch data by das catalog path, no server URLs required\n" +"\n" +"USAGE\n" +" " PROG " [options] roots\n" +" " PROG " [options] node [PATH]\n" +" " PROG " [options] json PATH[#FRAGMENT]\n" +" " PROG " [options] url PATH [KEY=VALUE ...]\n" +" " PROG " [options] data PATH [KEY=VALUE ...] > out_file\n" +" " PROG " COMMAND -h\n" +"\n" +"DESCRIPTION\n" +" The das federated catalog is a tree of small JSON documents. Catalog\n" +" nodes list other nodes, and data source nodes describe how to query a\n" +" server. " PROG " walks that tree so you never have to know where a\n" +" server lives or what its query string looks like.\n" +"\n" +" Interactive use is 'node' to look around, then 'data' to download. In a\n" +" script, 'data' writes the response body to standard output, which can be\n" +" redirected to a file or piped to das3_csv, das2_psd and friends. The\n" +" body is passed through untouched, so the output format is whatever the\n" +" source produces: das2 or das3 stream, CDF, CSV, or raw packets.\n" +"\n" +"COMMANDS\n" +" roots Print the compiled in catalog root URLs.\n" +" node [PATH] List a catalog's children, or a data source's examples\n" +" and query keys. No PATH shows the default root.\n" +" json PATH Print a node's raw JSON, or a fragment of it.\n" +" url PATH ... Print the HTTP GET URL a query turns into.\n" +" data PATH ... Run the query, write the response body to stdout.\n" +"\n" +" Each command has its own help: '" PROG " data -h' explains query keys,\n" +" '" PROG " node -h' explains the listings.\n" +"\n" +"PATHS\n" +" Paths are catalog paths, not web paths. Any of these forms work:\n" +"\n" +" uiowa/juno/wav/survey relative to the root prefix, by\n" +" default '" DEF_ROOT "'\n" +" site:/uiowa/juno/wav/survey das2py shorthand\n" +" tag:das2.org,2012:site:/uiowa full catalog path URI\n" +" https://host/some/node.json a node by direct URL\n" +" ./local_node.json a node file on disk\n" +"\n" +"OPTIONS\n" +" -h, --help This text, or with a command, that command's help.\n" +" -V, --version Print the program and library version and exit.\n" +"\n" +" -r, --root ROOT Where paths start, in one of three forms.\n" +" A catalog URI prefix for relative paths, default\n" +" '" DEF_ROOT "':\n" +" -r tag:das2.org,2012:test:/\n" +" A URL or file of a catalog node. It becomes the only\n" +" root, nothing outside it can be reached:\n" +" -r https://tracers-dev.physics.uiowa.edu/stream/root.json\n" +" -r ./my_catalog.json\n" +" URI=URL, the node at URL named URI, again as the only\n" +" root. Paths are relative to it unless they are full\n" +" URIs under it:\n" +" -r 10.1037/a0040251=https://doi.org/10.1037/a0040251/root.json\n" +" Without -r the environment variable " ROOT_ENV " is\n" +" used the same way, if set.\n" +" -g, --graft URI=URL\n" +" Patch the federated catalog: the node at URL stands in\n" +" for URI and everything under it, the rest of the\n" +" catalog is unchanged. Relative paths start at the\n" +" grafted node, or at the -r prefix if one is given:\n" +" -g site:/uiowa/tracers/tracers-1=\\\n" +" https://tracers-dev.physics.uiowa.edu/stream/source/tracers-1.json\n" +" The environment variable " GRAFT_ENV " stands in\n" +" for -g, if set.\n" +"\n" +" -s SECTIONS For 'node', which parts of a source to print, a comma\n" +" separated list from: examples, coords, data, formats,\n" +" options, all. The default leaves out formats.\n" +" -p PARAM=VALUE Send a raw HTTP GET parameter, bypassing the interface\n" +" translation. Repeatable.\n" +"\n" +" -f Fresh: ignore cached catalog nodes along the path and\n" +" fetch them again. Use after a catalog has changed.\n" +" -C DIR Keep cached catalog nodes in DIR instead of the\n" +" default below, or 'none' for no cache at all.\n" +"\n" +" -t SECONDS Connection timeout for the data request, default 10.\n" +" -q Quiet, only errors on stderr.\n" +" -v Verbose: node and server URLs, all source sections,\n" +" the arguments behind each example, usage hints, and\n" +" a report of each URL contacted.\n" +" -l LEVEL Log level: crit, error, warn, info, debug, trace.\n" +"\n" +"FILES\n" +" " HOME_VAR "/" DAS_DIR "/cache/nodes/\n" +" Catalog nodes fetched over HTTP, kept for an hour. Walking a catalog\n" +" costs one request per level, so repeat runs only talk to the data\n" +" server. See -f and -C.\n" +" " HOME_VAR "/" DAS_DIR "/" CRED_FILE "\n" +" Saved credentials. When a source needs them these are tried before\n" +" the username and password prompt.\n" +"\n" +"EXIT CODES\n" +" 0 success 2 usage error 3 path did not resolve 4 bad query\n" +" 5 network or server failure 6 authentication failed 7 write failed\n" +"\n" +"SEE ALSO\n" +" das3_csv, das3_text, das3_info, das2_psd\n" +"\n" +" https://das2.org, and 'json PATH' to see a node as the server sent it.\n" +"\n", stdout); +} + +static void prnHelpNode(void) +{ + fputs( +"USAGE\n" +" " PROG " [options] node [PATH]\n" +"\n" +" Show what is at a catalog path. With no PATH, the default root, which\n" +" lists the federated sites.\n" +"\n" +"CATALOGS\n" +" A catalog node prints its own name and URI, then one line per child:\n" +" name, node type, and title. Append a child's name to the path to step\n" +" down. The URI line ends with the separator this catalog uses, so the\n" +" next path is URI plus child name. A '*' after the type marks node\n" +" types " PROG " can list but not load or fetch from (Das2DSDF, WebSocSrc).\n" +"\n" +" A SourceSet or Collection holds several descriptions of one dataset.\n" +" Only the one 'data' and 'url' would use is shown: the child named\n" +" 'flex', else the first HttpStreamSrc. -v lists them all. To use\n" +" another, add its name to the path, as in uiowa/juno/wav/survey/das2.\n" +"\n" +"DATA SOURCES\n" +" An HttpStreamSrc node prints a header, its examples, then one table per\n" +" interface section.\n" +"\n" +" examples Key is the number to give 'data' or 'url' to run that\n" +" example. With -v the KEY=VALUE arguments behind each one\n" +" are shown too.\n" +" coords Coordinate properties, such as the time range.\n" +" data Data variables that can be switched on or off.\n" +" formats Output format groups and their settings. Hidden by\n" +" default, -s formats or -v shows them. The header line\n" +" 'Formats:' always lists the group names.\n" +" options Source options that fit nowhere else.\n" +"\n" +" Table columns: Key is what you type, without the section prefix; Type\n" +" and Units are the value's; Default is what the server uses when the key\n" +" is not given; Req marks keys the server insists on; Values lists an\n" +" enum's choices or a coordinate's range.\n" +"\n" +"OPTIONS\n" +" -s SECTIONS Print only these sections, comma separated: examples,\n" +" coords, data, formats, options, all.\n" +" -v Also print the node and server URLs, all sections, each\n" +" example's arguments, and usage hints.\n" +"\n" +"EXAMPLES\n" +" " PROG " node\n" +" " PROG " node uiowa/juno/wav\n" +" " PROG " node -s examples,coords uiowa/tracers/tracers-2/spice/geomagcoords\n" +" " PROG " -r https://tracers-dev.physics.uiowa.edu/stream/root.json node\n" +"\n", stdout); +} + +static void prnHelpQuery(bool bData) +{ + fprintf(stdout, +"USAGE\n" +" " PROG " [options] %s PATH [KEY=VALUE ...]\n" +"\n", bData ? "data" : "url "); + if(bData) + fputs( +" Resolve the query, request it, and write the response body to stdout\n" +" untouched, so the output is whatever the source produces. Diagnostics\n" +" go to stderr. A password prompt, if the source needs one, is read from\n" +" the terminal, not stdin. Exit code 0 means the whole body arrived.\n" +"\n", stdout); + else + fputs( +" Resolve the query and print the HTTP GET URL that 'data' would request,\n" +" without requesting it. For scripts, and for checking what a KEY=VALUE\n" +" pair turns into.\n" +"\n", stdout); + fputs( +"QUERY KEYS\n" +" Keys come from the source's interface definition, so they differ from\n" +" source to source. Run 'node PATH' to see them. They look like:\n" +"\n" +" coords.time.min=2026-03-01 a coordinate property\n" +" data.lshell.enabled=true a data variable property\n" +" formats.csv.delimiter=Tab an output format property\n" +" options.decompress=true a source option\n" +"\n" +" Shortcuts, as long as the result is unambiguous:\n" +"\n" +" time.min=2026-03-01 drop the section prefix\n" +" time=2026-03-01,2026-03-02 coordinate range MIN,MAX[,STEP]. STEP\n" +" is the resolution if the coordinate has\n" +" one, otherwise the interval. Leave an\n" +" element empty to skip it: time=,2026-03-02\n" +" lshell=true or just lshell toggle a data variable or boolean option\n" +" csv or formats.csv select an output format group\n" +" 1 or examples.1 start from the maintainer's example 1,\n" +" then apply the other keys on top\n" +" min / minimum, max / maximum, res / resolution, inter / interval are\n" +" interchangeable property names\n" +"\n" +" With no keys at all, example 0 runs, so a bare 'data PATH' always\n" +" produces something to look at.\n" +"\n" +" Keys set to their default value are not sent to the server, except for\n" +" parameters the server requires. Required parameters without a default\n" +" must be given or the request is refused locally with a message naming\n" +" the key to set. Nothing in these rules assumes the first coordinate is\n" +" time; the coordinate names are whatever the source declares.\n" +"\n" +" A source that offers both stream versions has a key for it, usually\n" +" version=das2 or version=das3; its default is what the server sends when\n" +" you say nothing.\n" +"\n" +"OPTIONS\n" +" -p PARAM=VALUE Send a raw HTTP GET parameter, bypassing the interface\n" +" translation. Repeatable. Ties your script to one\n" +" server's API, so prefer KEY=VALUE when you can.\n" +" -t SECONDS Connection timeout, default 10.\n" +"\n" +"EXAMPLES\n" +" A day of Juno Waves survey data:\n" +"\n" +" " PROG " data uiowa/juno/wav/survey time=2021-02-21,2021-02-22 \\\n" +" > jno_wav_2021-02-21.d2s\n" +"\n" +" A source's first example, just to see that it works:\n" +"\n" +" " PROG " data uiowa/tracers/tracers-2/spice/geomagcoords | das3_text -3\n" +"\n" +" Predicted TRACERS-2 magnetic coordinates, one point a minute, as CSV:\n" +"\n" +" " PROG " data uiowa/tracers/tracers-2/spice/geomagcoords \\\n" +" time=2026-04-01,2026-04-01T01:30,60 csv\n" +"\n" +" One housekeeping value from a development server, through das3_csv:\n" +"\n" +" " PROG " -r https://tracers-dev.physics.uiowa.edu/stream/root.json \\\n" +" data tracers-2/ql/mag/hsk time=2026-01-01,2026-01-02 sprt | das3_csv\n" +"\n" +" The URL a query turns into, without running it:\n" +"\n" +" " PROG " url uiowa/juno/wav/survey time=2021-02-21,2021-02-22 b\n" +"\n", stdout); +} + +static void prnHelpJson(void) +{ + fputs( +"USAGE\n" +" " PROG " [options] json PATH[#FRAGMENT]\n" +"\n" +" Print the node's JSON document, pretty printed, or just the part at\n" +" FRAGMENT, a slash separated path into it. For looking at what the\n" +" catalog really says when a listing leaves you wondering.\n" +"\n" +"EXAMPLES\n" +" " PROG " json uiowa/juno/wav/survey\n" +" " PROG " json 'uiowa/tracers/tracers-2/spice/geomagcoords#protocol/httpParams'\n" +" " PROG " json 'uiowa/tracers/tracers-2/spice/geomagcoords#interface/examples'\n" +"\n", stdout); +} + +static void prnHelpRoots(void) +{ + fputs( +"USAGE\n" +" " PROG " roots\n" +"\n" +" Print the compiled in URLs of the federated catalog root, one per line.\n" +" The root is mirrored, the first URL that answers is used. A -r ROOT\n" +" argument or the " ROOT_ENV " variable replaces it, a -g URI=URL argument\n" +" or the " GRAFT_ENV " variable patches it, see '" PROG " -h'.\n" +"\n", stdout); +} + +/* Help for a command, or the overview */ +static void prnHelp(const char* sCmd) +{ + if(sCmd == NULL) prnHelpMain(); + else if((strcmp(sCmd, "node") == 0)||(strcmp(sCmd, "ls") == 0)) prnHelpNode(); + else if((strcmp(sCmd, "data") == 0)||(strcmp(sCmd, "get") == 0)) prnHelpQuery(true); + else if(strcmp(sCmd, "url") == 0) prnHelpQuery(false); + else if(strcmp(sCmd, "json") == 0) prnHelpJson(); + else if(strcmp(sCmd, "roots") == 0) prnHelpRoots(); + else prnHelpMain(); +} + +/* ************************************************************************* */ +/* Small string and JSON helpers */ + +static bool strieq(const char* a, const char* b) +{ + if((a == NULL)||(b == NULL)) return false; + while(*a && *b){ + if(tolower((unsigned char)*a) != tolower((unsigned char)*b)) return false; + ++a; ++b; + } + return (*a == '\0') && (*b == '\0'); +} + +static bool strpfx(const char* s, const char* sPrefix) +{ + return (s != NULL) && (strncmp(s, sPrefix, strlen(sPrefix)) == 0); +} + +static void strcpyz(char* sDest, const char* sSrc, size_t uDest) +{ + if(uDest == 0) return; + if(sSrc == NULL){ sDest[0] = '\0'; return; } + strncpy(sDest, sSrc, uDest - 1); + sDest[uDest - 1] = '\0'; +} + +static bool parseBool(const char* s, bool* pOut) +{ + if(s == NULL) return false; + if(strieq(s,"true")||strieq(s,"yes")||strieq(s,"on")||strieq(s,"1")|| + strieq(s,"t")||strieq(s,"y")){ *pOut = true; return true; } + if(strieq(s,"false")||strieq(s,"no")||strieq(s,"off")||strieq(s,"0")|| + strieq(s,"f")||strieq(s,"n")){ *pOut = false; return true; } + return false; +} + +/* NULL tolerant wrappers. DasJdo_get() is a slash path lookup, so the dot + * in HTTP parameter names such as "read.time.min" is not special to it. */ +static const DasJdo* jget(const DasJdo* p, const char* sPath) +{ + return (p == NULL) ? NULL : DasJdo_get(p, sPath); +} + +static const char* jstr(const DasJdo* p, const char* sPath) +{ + return DasJdo_string(jget(p, sPath)); +} + +static bool jisdict(const DasJdo* p){ return (p != NULL)&&(p->type == das_json_type_dict); } +static bool jisary(const DasJdo* p){ return (p != NULL)&&(p->type == das_json_type_ary); } +static bool jtrue(const DasJdo* p){ return (p != NULL)&&(p->type == das_json_type_true); } +static bool jisbool(const DasJdo* p){ + return (p != NULL)&&((p->type == das_json_type_true)||(p->type == das_json_type_false)); +} + +/* Scalar JSON value to text. False for null, missing, dicts and arrays. + * Example settings and defaults can be strings, numbers or booleans, and the + * value sent to the server is text in any case. */ +static bool jscalar(const DasJdo* p, char* sBuf, size_t uLen) +{ + if((p == NULL)||(uLen == 0)) return false; + switch(p->type){ + case das_json_type_str: + strcpyz(sBuf, ((const das_json_str*)p->value)->string, uLen); + return true; + case das_json_type_num:{ + const das_json_num* pNum = (const das_json_num*)p->value; + size_t u = (pNum->number_size < uLen - 1) ? pNum->number_size : uLen - 1; + memcpy(sBuf, pNum->number, u); + sBuf[u] = '\0'; + return true; + } + case das_json_type_true: strcpyz(sBuf, "true", uLen); return true; + case das_json_type_false: strcpyz(sBuf, "false", uLen); return true; + default: return false; + } +} + +/* Percent encode everything but RFC 3986 unreserved characters. Servers + * decode before use, so over-encoding ':' in time strings is harmless and + * matches what das2py's quote_plus does. */ +static void urlEncode(const char* sIn, char* sOut, size_t uOut) +{ + static const char* sHex = "0123456789ABCDEF"; + size_t u = 0; + for(const unsigned char* p = (const unsigned char*)sIn; *p && (u + 4 < uOut); ++p){ + if(isalnum(*p)||(*p == '-')||(*p == '.')||(*p == '_')||(*p == '~')){ + sOut[u++] = (char)*p; + } + else{ + sOut[u++] = '%'; + sOut[u++] = sHex[*p >> 4]; + sOut[u++] = sHex[*p & 0x0F]; + } + } + sOut[u] = '\0'; +} + +/* ************************************************************************* */ +/* The settable model: a flat list of interface properties that have a 'set' + * block, with enough context to print them and to translate values. + * + * DasHttpSrc, DasSettable and DasHttpParam are named for where they are + * expected to end up, libdas3 next to DasNode, once the 1.0 catalog schema + * settles what they read (see cf_notes/api_enlargement.md). The _print and + * _describe functions are das_fetch presentation and would stay behind. */ + +typedef enum { SEC_COORD = 0, SEC_DATA, SEC_FORMAT, SEC_OPTION, SEC_COUNT } sec_e; + +/* Section names as the 0.7 catalogs spell them. Keys are built from these, + input accepts the singular forms too, see secAlias() */ +static const char* g_sSecName[SEC_COUNT] = {"coords", "data", "formats", "options"}; + +typedef struct das_settable { + sec_e sec; + char sGroup[NAME_SZ]; /* "time", "lshell", "das"; empty for options */ + char sProp[NAME_SZ]; /* "min", "enabled", "serial" */ + char sKey[KEY_SZ]; /* "coord.time.min" */ + const DasJdo* pGroup; /* the coordinate, variable, or format dict */ + const DasJdo* pProp; /* the property dict: value, set, type, label... */ + const DasJdo* pSet; /* the set dict: param, flag, pval, enum... */ + const char* sParam; /* set/param, the HTTP GET parameter name */ + const DasJdo* pHParam; /* protocol httpParams entry for sParam */ + bool bRequired; /* set/required or httpParams/required */ + bool bHasDefault; /* value is not null */ + char sDefault[VAL_SZ]; + bool bUserSet; + char sValue[VAL_SZ]; +} DasSettable; + +typedef struct das_http_param { + char sName[NAME_SZ]; + char sVal[VAL_SZ]; +} DasHttpParam; + +typedef struct das_http_src { + DasNode* pNode; + const DasJdo* pIface; + const DasJdo* pProto; + const DasJdo* pHttpParams; + const DasJdo* pBaseUrls; + bool bLegacy; /* v0.6 layout */ + DasSettable aSet[MAX_SETTABLES]; + size_t uSet; + DasHttpParam aOut[MAX_PARAMS]; + size_t uOut; +} DasHttpSrc; + +/* Property name aliases. The 0.6 nodes spell out minimum/maximum/resolution + * and interval, 0.7 nodes abbreviate. Users may type either. */ +static const char* canonProp(const char* s) +{ + if(strieq(s, "minimum")||strieq(s, "min")) return "min"; + if(strieq(s, "maximum")||strieq(s, "max")) return "max"; + if(strieq(s, "resolution")||strieq(s, "res")) return "res"; + if(strieq(s, "interval")||strieq(s, "inter")) return "inter"; + if(strieq(s, "enable")||strieq(s, "enabled")) return "enabled"; + return s; +} + +static bool propEq(const char* a, const char* b) +{ + return strieq(canonProp(a), canonProp(b)); +} + +static bool secAlias(const char* s, sec_e* pSec) +{ + if(strieq(s,"coord")||strieq(s,"coords")||strieq(s,"coordinate")||strieq(s,"coordinates")){ + *pSec = SEC_COORD; return true; + } + if(strieq(s,"data")){ *pSec = SEC_DATA; return true; } + if(strieq(s,"format")||strieq(s,"formats")||strieq(s,"fmt")){ *pSec = SEC_FORMAT; return true; } + if(strieq(s,"option")||strieq(s,"options")||strieq(s,"opt")||strieq(s,"opts")){ + *pSec = SEC_OPTION; return true; + } + return false; +} + +/* ************************************************************************* */ +/* Display settings and terminal wrapping */ + +/* Which parts of a source 'node' prints: one bit per section plus examples */ +#define SHOW_EXAMPLES (1u << SEC_COUNT) +#define SHOW_ALL ((1u << (SEC_COUNT + 1)) - 1) +static unsigned g_uShow = SHOW_ALL & ~(1u << SEC_FORMAT); +static bool g_bVerbose = false; +static int g_nCols = 0; /* terminal width, 0 when stdout is not a terminal */ + +/* Display width of UTF-8 text, code points not bytes. Catalog text has no + * double width characters in practice, so this is as far as it goes. */ +static size_t u8width(const char* s, size_t uBytes) +{ + size_t w = 0; + for(size_t u = 0; u < uBytes; ++u) if((((unsigned char)s[u]) & 0xC0) != 0x80) ++w; + return w; +} + +/* Print text that starts at column nCol, breaking on spaces at the terminal + * width with continuation lines indented back to nCol. Without a terminal, + * or with too little room to be worth it, the text goes out as is. */ +static void fputsWrappedAt(const char* sText, int nCol, int nWidth, FILE* pOut) +{ + int nAvail = nWidth - nCol; + if((nWidth <= 0)||(nAvail < 20)||(u8width(sText, strlen(sText)) <= (size_t)nAvail)){ + fputs(sText, pOut); + return; + } + const char* p = sText; + bool bFirst = true; + while(*p != '\0'){ + while(*p == ' ') ++p; + if(*p == '\0') break; + const char* pEnd = p; + const char* pBreak = NULL; + size_t w = 0; + while((*pEnd != '\0') && (w < (size_t)nAvail)){ + if(*pEnd == ' ') pBreak = pEnd; + if((((unsigned char)*pEnd) & 0xC0) != 0x80) ++w; + ++pEnd; + } + if((*pEnd == '\0')||(*pEnd == ' ')) pBreak = pEnd; /* fits, or breaks right here */ + else if(pBreak == NULL) pBreak = pEnd; /* one long word, cut it */ + if(!bFirst){ + fputc('\n', pOut); + for(int i = 0; i < nCol; ++i) fputc(' ', pOut); + } + fwrite(p, 1, (size_t)(pBreak - p), pOut); + bFirst = false; + p = pBreak; + } +} + +static void fputsWrapped(const char* sText, int nCol, FILE* pOut) +{ + fputsWrappedAt(sText, nCol, g_nCols, pOut); +} + +/* A paragraph of explanation, wrapped to the terminal, or to 80 columns + * when there is none so piped output does not get one endless line. An + * optional indent applies to every line. */ +static void prnPara(FILE* pOut, int nIndent, const char* sText) +{ + for(int i = 0; i < nIndent; ++i) fputc(' ', pOut); + fputsWrappedAt(sText, nIndent, (g_nCols > 0) ? g_nCols : 80, pOut); + fputc('\n', pOut); +} + +/* A labeled header line, wrapped to the terminal */ +static void prnField(FILE* pOut, const char* sLabel, const char* sText) +{ + if(sText == NULL) return; + fprintf(pOut, "%-10s", sLabel); + fputsWrapped(sText, 10, pOut); + fputc('\n', pOut); +} + +/* Type of a settable for validation and display. Preference order: the + * property's own type, an enum in set, a boolean default, a flag toggle, + * the protocol parameter's type, then string. */ +static const char* DasSettable_type(const DasSettable* p) +{ + const char* sType = jstr(p->pProp, "type"); + if(sType != NULL){ + if(strieq(sType, "bool")) return "boolean"; + return sType; + } + if(jisary(jget(p->pSet, "enum"))) return "enum"; + if(jisbool(jget(p->pProp, "value"))) return "boolean"; + if(jstr(p->pSet, "flag") != NULL) return "boolean"; + sType = jstr(p->pHParam, "type"); + if(sType != NULL){ + if(strieq(sType, "FlagSet")||strieq(sType, "flag_set")) return "string"; + return sType; + } + return "string"; +} + +static bool DasSettable_isBool(const DasSettable* p) +{ + return strieq(DasSettable_type(p), "boolean"); +} + +/* Units for display. 0.7 puts a units string on the property; both versions + * may carry a units dict with a "value" at the group level (0.7 under props, + * 0.6 directly). */ +static const char* DasSettable_units(const DasSettable* p) +{ + const DasJdo* pU = jget(p->pProp, "units"); + if(pU != NULL){ + if(pU->type == das_json_type_str) return DasJdo_string(pU); + if(jstr(pU, "value")) return jstr(pU, "value"); + } + if(p->sec == SEC_COORD){ + const char* s = jstr(p->pGroup, "props/units/value"); + if(s == NULL) s = jstr(p->pGroup, "units/value"); + if(s != NULL) return s; + } + if(strieq(DasSettable_type(p), "isotime")) return "UTC"; + return ""; +} + +static const char* groupName(const DasSettable* p) +{ + const char* s = jstr(p->pGroup, "label"); + if(s == NULL) s = jstr(p->pGroup, "name"); + if(s == NULL) s = jstr(p->pGroup, "title"); + return s ? s : p->sGroup; +} + +/* Description column. The property's own title or label wins. Without + * one, coordinate properties have conventional meanings worth spelling out, + * and an enable toggle is best described by the thing it enables. */ +static const char* DasSettable_desc(const DasSettable* p, char* sBuf, size_t uLen) +{ + const char* s = jstr(p->pProp, "title"); + if(s == NULL) s = jstr(p->pProp, "summary"); + if(s == NULL) s = jstr(p->pProp, "label"); + if(s == NULL) s = jstr(p->pProp, "name"); + if(s != NULL) return s; + + const char* sProp = canonProp(p->sProp); + if(strcmp(sProp, "enabled") == 0){ + if(p->sec == SEC_FORMAT){ + snprintf(sBuf, uLen, "Output as %s", groupName(p)); + } + else{ + const char* sTitle = jstr(p->pGroup, "title"); + if(sTitle == NULL) sTitle = groupName(p); + snprintf(sBuf, uLen, "Output %s", sTitle); + } + return sBuf; + } + if(p->sec == SEC_COORD){ + if(strcmp(sProp, "min") == 0){ + snprintf(sBuf, uLen, "Minimum %s value to output", groupName(p)); + return sBuf; + } + if(strcmp(sProp, "max") == 0){ + snprintf(sBuf, uLen, "Maximum %s value to output", groupName(p)); + return sBuf; + } + if(strcmp(sProp, "res") == 0){ + snprintf(sBuf, uLen, "Average over %s bins of this width", groupName(p)); + return sBuf; + } + if(strcmp(sProp, "inter") == 0){ + snprintf(sBuf, uLen, "Step between output %s values", groupName(p)); + return sBuf; + } + } + if(p->sec != SEC_OPTION){ + snprintf(sBuf, uLen, "%s %s", groupName(p), p->sProp); + return sBuf; + } + return ""; +} + +/* Allowed values for display: enum entries, a range, or a coordinate's + * valid range for min/max properties. */ +static void DasSettable_values(const DasSettable* p, char* sBuf, size_t uLen) +{ + sBuf[0] = '\0'; + const DasJdo* pEnum = jget(p->pSet, "enum"); + if(jisary(pEnum)){ + size_t u = 0; + char sVal[VAL_SZ]; + for(const das_json_ary_el* pEl = DasJdo_aryFirst(pEnum); pEl; pEl = pEl->next){ + if(!jscalar(jget(pEl->value, "value"), sVal, sizeof(sVal))) continue; + if(u + strlen(sVal) + 3 >= uLen){ strcpyz(sBuf + u, "...", uLen - u); return; } + u += snprintf(sBuf + u, uLen - u, "%s%s", (u > 0) ? ", " : "", sVal); + } + return; + } + const DasJdo* pRange = jget(p->pSet, "range"); + if(!jisary(pRange)) pRange = jget(p->pHParam, "range"); + if(!jisary(pRange) && (p->sec == SEC_COORD) && + (propEq(p->sProp, "min") || propEq(p->sProp, "max"))) + pRange = jget(p->pGroup, "validRange"); + if(jisary(pRange)){ + const das_json_ary_el* pEl = DasJdo_aryFirst(pRange); + char sLo[VAL_SZ] = {'\0'}, sHi[VAL_SZ] = {'\0'}; + if(pEl) jscalar(pEl->value, sLo, sizeof(sLo)); + if(pEl && pEl->next) jscalar(pEl->next->value, sHi, sizeof(sHi)); + snprintf(sBuf, uLen, "%s to %s", sLo, sHi); + return; + } + if(DasSettable_isBool(p)){ + strcpyz(sBuf, "true, false", uLen); + } +} + +/* ************************************************************************* */ +/* Flattening the interface */ + +static void DasHttpSrc_addProp( + DasHttpSrc* pS, sec_e sec, const char* sGroup, const DasJdo* pGroup, + const char* sProp, const DasJdo* pProp +){ + if(!jisdict(pProp)) return; + const DasJdo* pSet = jget(pProp, "set"); + if(!jisdict(pSet)) return; /* documentation only, not settable */ + + if(pS->uSet >= MAX_SETTABLES){ + daslog_warn_v("Source lists more than %d settable properties, ignoring %s", + MAX_SETTABLES, sProp); + return; + } + DasSettable* p = pS->aSet + pS->uSet; + memset(p, 0, sizeof(DasSettable)); + pS->uSet += 1; + + p->sec = sec; + strcpyz(p->sGroup, sGroup, NAME_SZ); + strcpyz(p->sProp, sProp, NAME_SZ); + if(sec == SEC_OPTION) + snprintf(p->sKey, KEY_SZ, "%s.%s", g_sSecName[sec], sProp); + else + snprintf(p->sKey, KEY_SZ, "%s.%s.%s", g_sSecName[sec], sGroup, sProp); + + p->pGroup = pGroup; + p->pProp = pProp; + p->pSet = pSet; + p->sParam = jstr(pSet, "param"); + p->pHParam = p->sParam ? jget(pS->pHttpParams, p->sParam) : NULL; + p->bRequired = jtrue(jget(pSet, "required")) || jtrue(jget(p->pHParam, "required")); + p->bHasDefault = jscalar(jget(pProp, "value"), p->sDefault, VAL_SZ); +} + +/* A group is a coordinate, a data variable, a format, or the whole options + * block. 0.7 groups keep properties under "props", 0.6 groups hold them + * directly beside label/title/units. Anything without a 'set' is skipped + * either way, so iterating the group itself is safe for 0.6. */ +static void DasHttpSrc_addGroup(DasHttpSrc* pS, sec_e sec, const char* sGroup, const DasJdo* pGroup) +{ + if(!jisdict(pGroup)) return; + const DasJdo* pProps = jget(pGroup, "props"); + const DasJdo* pIter = jisdict(pProps) ? pProps : pGroup; + for(const das_json_dict_el* pEl = DasJdo_dictFirst(pIter); pEl; pEl = pEl->next) + DasHttpSrc_addProp(pS, sec, sGroup, pGroup, pEl->name->string, pEl->value); +} + +static void DasHttpSrc_addSection(DasHttpSrc* pS, sec_e sec, const DasJdo* pSec) +{ + if(!jisdict(pSec)) return; + for(const das_json_dict_el* pEl = DasJdo_dictFirst(pSec); pEl; pEl = pEl->next) + DasHttpSrc_addGroup(pS, sec, pEl->name->string, pEl->value); +} + +static bool DasHttpSrc_init(DasHttpSrc* pS, DasNode* pNode, char* sErr, size_t uErr) +{ + memset(pS, 0, sizeof(DasHttpSrc)); + pS->pNode = pNode; + const DasJdo* pDom = DasNode_getJdo(pNode, NULL); + pS->pIface = jget(pDom, "interface"); + pS->pProto = jget(pDom, "protocol"); + if(!jisdict(pS->pProto)){ + snprintf(sErr, uErr, "Source node %s has no 'protocol' section", DasNode_srcUrl(pNode)); + return false; + } + pS->pHttpParams = jget(pS->pProto, "httpParams"); + if(pS->pHttpParams == NULL) pS->pHttpParams = jget(pS->pProto, "http_params"); + pS->pBaseUrls = jget(pS->pProto, "baseUrls"); + if(pS->pBaseUrls == NULL) pS->pBaseUrls = jget(pS->pProto, "base_urls"); + if(!jisary(pS->pBaseUrls)){ + snprintf(sErr, uErr, "Source node %s has no base URL list", DasNode_srcUrl(pNode)); + return false; + } + pS->bLegacy = (jget(pS->pProto, "http_params") != NULL) || + (jget(pS->pIface, "coordinates") != NULL); + + const DasJdo* pSec = jget(pS->pIface, "coords"); + if(pSec == NULL) pSec = jget(pS->pIface, "coordinates"); + DasHttpSrc_addSection(pS, SEC_COORD, pSec); + DasHttpSrc_addSection(pS, SEC_DATA, jget(pS->pIface, "data")); + DasHttpSrc_addSection(pS, SEC_FORMAT, jget(pS->pIface, "formats")); + DasHttpSrc_addGroup(pS, SEC_OPTION, "", jget(pS->pIface, "options")); + return true; +} + +/* ************************************************************************* */ +/* Key lookup */ + +/* Split "a.b.c" into parts, returns the count or 0 if malformed */ +static int splitKey(const char* sKey, char aParts[3][NAME_SZ]) +{ + int n = 0; + const char* pBeg = sKey; + while(true){ + const char* pEnd = strchr(pBeg, '.'); + size_t uLen = pEnd ? (size_t)(pEnd - pBeg) : strlen(pBeg); + if((uLen == 0)||(uLen >= NAME_SZ)||(n == 3)) return 0; + memcpy(aParts[n], pBeg, uLen); + aParts[n][uLen] = '\0'; + ++n; + if(pEnd == NULL) break; + pBeg = pEnd + 1; + } + return n; +} + +/* Find settables matching a dotted key. Returns the match count and fills + * ppOut with up to nMax of them. More than one match is an error the caller + * reports with the candidate list; zero means no such key. */ +static int DasHttpSrc_match(DasHttpSrc* pS, const char* sKey, DasSettable** ppOut, int nMax) +{ + char aParts[3][NAME_SZ]; + int nParts = splitKey(sKey, aParts); + if(nParts == 0) return 0; + + int n = 0; + sec_e sec; + DasSettable* p = NULL; + +#define _ADD_MATCH if(n < nMax) ppOut[n] = p; ++n; + + if(nParts == 3){ + if(!secAlias(aParts[0], &sec)) return 0; + for(size_t u = 0; u < pS->uSet; ++u){ + p = pS->aSet + u; + if((p->sec == sec) && strieq(p->sGroup, aParts[1]) && propEq(p->sProp, aParts[2])){ + _ADD_MATCH + } + } + return n; + } + + if(nParts == 2){ + /* "option.X" is a section qualified option, "SEC.GROUP" toggles a + variable, anything else is GROUP.PROP in any section */ + if(secAlias(aParts[0], &sec)){ + for(size_t u = 0; u < pS->uSet; ++u){ + p = pS->aSet + u; + if(p->sec != sec) continue; + if((sec == SEC_OPTION) && propEq(p->sProp, aParts[1])){ _ADD_MATCH } + else if((sec != SEC_OPTION) && strieq(p->sGroup, aParts[1]) && + propEq(p->sProp, "enabled")){ _ADD_MATCH } + } + if(n > 0) return n; + } + for(size_t u = 0; u < pS->uSet; ++u){ + p = pS->aSet + u; + if(strieq(p->sGroup, aParts[0]) && propEq(p->sProp, aParts[1])){ _ADD_MATCH } + } + return n; + } + + /* One part: an option, or a variable/format toggle */ + for(size_t u = 0; u < pS->uSet; ++u){ + p = pS->aSet + u; + if((p->sec == SEC_OPTION) && propEq(p->sProp, aParts[0])){ _ADD_MATCH } + else if((p->sec != SEC_OPTION) && strieq(p->sGroup, aParts[0]) && + propEq(p->sProp, "enabled")){ _ADD_MATCH } + } + if(n > 0) return n; + + /* Last resort, a bare property name. Unique in practice for things + like 'serial', ambiguous for 'fracSecs' shared by das and csv, and + the ambiguity message then lists the full keys. */ + for(size_t u = 0; u < pS->uSet; ++u){ + p = pS->aSet + u; + if((p->sec != SEC_OPTION) && propEq(p->sProp, aParts[0])){ _ADD_MATCH } + } +#undef _ADD_MATCH + return n; +} + +/* If sKey (with or without a coord. prefix) names a coordinate group that + * has settable properties, return the group name as spelled by the node */ +static const char* DasHttpSrc_coordGroup(DasHttpSrc* pS, const char* sKey) +{ + char aParts[3][NAME_SZ]; + int nParts = splitKey(sKey, aParts); + const char* sGroup = NULL; + sec_e sec; + if(nParts == 1) sGroup = aParts[0]; + else if((nParts == 2) && secAlias(aParts[0], &sec) && (sec == SEC_COORD)) sGroup = aParts[1]; + else return NULL; + + for(size_t u = 0; u < pS->uSet; ++u){ + if((pS->aSet[u].sec == SEC_COORD) && strieq(pS->aSet[u].sGroup, sGroup)) + return pS->aSet[u].sGroup; + } + return NULL; +} + +static bool DasHttpSrc_hasProp(const DasHttpSrc* pS, sec_e sec, const char* sGroup, const char* sProp) +{ + for(size_t u = 0; u < pS->uSet; ++u){ + const DasSettable* p = pS->aSet + u; + if((p->sec == sec) && strieq(p->sGroup, sGroup) && propEq(p->sProp, sProp)) + return true; + } + return false; +} + +static DasSettable* DasHttpSrc_find(DasHttpSrc* pS, sec_e sec, const char* sGroup, const char* sProp) +{ + for(size_t u = 0; u < pS->uSet; ++u){ + DasSettable* p = pS->aSet + u; + if((p->sec == sec) && strieq(p->sGroup, sGroup) && propEq(p->sProp, sProp)) + return p; + } + return NULL; +} + +/* Validate and store one user value. */ +static bool DasSettable_set(DasSettable* p, const char* sVal, char* sErr, size_t uErr) +{ + const char* sType = DasSettable_type(p); + + if(strieq(sType, "boolean")){ + bool b; + if(!parseBool(sVal, &b)){ + snprintf(sErr, uErr, "'%s' is not a boolean value for %s, use true or false", + sVal, p->sKey); + return false; + } + strcpyz(p->sValue, b ? "true" : "false", VAL_SZ); + p->bUserSet = true; + return true; + } + + const DasJdo* pEnum = jget(p->pSet, "enum"); + if(jisary(pEnum)){ + char sItem[VAL_SZ]; + for(const das_json_ary_el* pEl = DasJdo_aryFirst(pEnum); pEl; pEl = pEl->next){ + if(jscalar(jget(pEl->value, "value"), sItem, sizeof(sItem)) && strieq(sItem, sVal)){ + strcpyz(p->sValue, sItem, VAL_SZ); /* node's spelling */ + p->bUserSet = true; + return true; + } + } + char sVals[ERR_SZ/2]; + DasSettable_values(p, sVals, sizeof(sVals)); + snprintf(sErr, uErr, "'%s' is not an allowed value for %s, choose one of: %s", + sVal, p->sKey, sVals); + return false; + } + + strcpyz(p->sValue, sVal, VAL_SZ); + p->bUserSet = true; + return true; +} + +/* Apply one KEY=VALUE from the command line, including the shortcuts. */ +static bool DasHttpSrc_set(DasHttpSrc* pS, const char* sKey, const char* sVal, char* sErr, size_t uErr) +{ + /* Coordinate range shortcut: GROUP=MIN,MAX[,STEP] */ + const char* sGroup = DasHttpSrc_coordGroup(pS, sKey); + if(sGroup != NULL){ + const char* aProps[3] = {"min", "max", NULL}; + aProps[2] = DasHttpSrc_hasProp(pS, SEC_COORD, sGroup, "res") ? "res" : "inter"; + + char sCopy[VAL_SZ]; + strcpyz(sCopy, sVal, VAL_SZ); + char* pTok = sCopy; + for(int i = 0; i < 4; ++i){ + char* pComma = strchr(pTok, ','); + if(pComma) *pComma = '\0'; + if(i == 3){ + snprintf(sErr, uErr, "Too many values for coordinate %s, at most MIN,MAX,STEP", sGroup); + return false; + } + if(pTok[0] != '\0'){ + DasSettable* p = DasHttpSrc_find(pS, SEC_COORD, sGroup, aProps[i]); + if(p == NULL){ + snprintf(sErr, uErr, "Coordinate %s has no settable '%s' property", sGroup, aProps[i]); + return false; + } + if(!DasSettable_set(p, pTok, sErr, uErr)) return false; + } + if(pComma == NULL) break; + pTok = pComma + 1; + } + return true; + } + + DasSettable* aMatch[8]; + int n = DasHttpSrc_match(pS, sKey, aMatch, 8); + if(n == 0){ + snprintf(sErr, uErr, "No query key named '%s' for this source, run '" PROG + " node PATH' to list them", sKey); + return false; + } + if(n > 1){ + size_t u = snprintf(sErr, uErr, "Key '%s' is ambiguous, use one of:", sKey); + for(int i = 0; (i < n) && (i < 8) && (u < uErr); ++i) + u += snprintf(sErr + u, uErr - u, " %s", aMatch[i]->sKey); + return false; + } + return DasSettable_set(aMatch[0], sVal, sErr, uErr); +} + +/* ************************************************************************* */ +/* Translation to protocol parameters */ + +static DasHttpParam* DasHttpSrc_outFind(DasHttpSrc* pS, const char* sName) +{ + for(size_t u = 0; u < pS->uOut; ++u) + if(strcmp(pS->aOut[u].sName, sName) == 0) return pS->aOut + u; + return NULL; +} + +static bool DasHttpSrc_outSet(DasHttpSrc* pS, const char* sName, const char* sVal, char* sErr, size_t uErr) +{ + DasHttpParam* p = DasHttpSrc_outFind(pS, sName); + if(p == NULL){ + if(pS->uOut >= MAX_PARAMS){ + snprintf(sErr, uErr, "More than %d HTTP parameters in one query", MAX_PARAMS); + return false; + } + p = pS->aOut + pS->uOut; + pS->uOut += 1; + strcpyz(p->sName, sName, NAME_SZ); + p->sVal[0] = '\0'; + } + strcpyz(p->sVal, sVal, VAL_SZ); + return true; +} + +/* Flag set parameters accumulate: several interface toggles map to one + * parameter whose value is the flags joined by a separator */ +static bool DasHttpSrc_outAddFlag( + DasHttpSrc* pS, const char* sName, const char* sFlagVal, const char* sSep, + char* sErr, size_t uErr +){ + DasHttpParam* p = DasHttpSrc_outFind(pS, sName); + if((p == NULL)||(p->sVal[0] == '\0')) + return DasHttpSrc_outSet(pS, sName, sFlagVal, sErr, uErr); + if(strlen(p->sVal) + strlen(sSep) + strlen(sFlagVal) >= VAL_SZ){ + snprintf(sErr, uErr, "Value for HTTP parameter %s is too long", sName); + return false; + } + strcat(p->sVal, sSep); + strcat(p->sVal, sFlagVal); + return true; +} + +static bool DasHttpSrc_translate(DasHttpSrc* pS, DasSettable* p, const char* sVal, char* sErr, size_t uErr) +{ + if(p->sParam == NULL){ + snprintf(sErr, uErr, "Key %s has no 'param' in its set block, the node is broken", p->sKey); + return false; + } + + char sFlag[NAME_SZ] = {'\0'}; + char sPVal[VAL_SZ] = {'\0'}; + bool bHavePVal = false; + strcpyz(sFlag, jstr(p->pSet, "flag"), NAME_SZ); + bHavePVal = jscalar(jget(p->pSet, "pval"), sPVal, VAL_SZ); + + const DasJdo* pEnum = jget(p->pSet, "enum"); + if(jisary(pEnum)){ + /* DasSettable_set already checked membership, now pick up per-item + overrides of the protocol value or flag */ + char sItem[VAL_SZ]; + for(const das_json_ary_el* pEl = DasJdo_aryFirst(pEnum); pEl; pEl = pEl->next){ + if(!jscalar(jget(pEl->value, "value"), sItem, sizeof(sItem))) continue; + if(!strieq(sItem, sVal)) continue; + if(jscalar(jget(pEl->value, "pval"), sPVal, VAL_SZ)) bHavePVal = true; + if(jstr(pEl->value, "flag")) strcpyz(sFlag, jstr(pEl->value, "flag"), NAME_SZ); + break; + } + } + else if(DasSettable_isBool(p)){ + bool bUser = false; + parseBool(sVal, &bUser); + const DasJdo* pTrig = jget(p->pSet, "value"); + if(jisbool(pTrig)){ + /* The set block fires for one state only, usually true. The other + state is the default and has nothing to send. */ + if(bUser != jtrue(pTrig)) return true; + } + if(!bHavePVal && (sFlag[0] == '\0')){ + strcpyz(sPVal, bUser ? "true" : "false", VAL_SZ); + bHavePVal = true; + } + } + + if(sFlag[0] != '\0'){ + char sPath[NAME_SZ + 8]; + snprintf(sPath, sizeof(sPath), "flags/%s", sFlag); + const DasJdo* pF = jget(p->pHParam, sPath); + if(pF == NULL){ + snprintf(sErr, uErr, "Flag '%s' for key %s is not defined under protocol " + "parameter %s, the node is broken", sFlag, p->sKey, p->sParam); + return false; + } + char sFVal[VAL_SZ]; + if(!jscalar(jget(pF, "value"), sFVal, VAL_SZ)) strcpyz(sFVal, sFlag, VAL_SZ); + const char* sPrefix = jstr(pF, "prefix"); + if(sPrefix != NULL){ + char sTmp[VAL_SZ]; + snprintf(sTmp, VAL_SZ, "%s%s", sPrefix, sFVal); + strcpyz(sFVal, sTmp, VAL_SZ); + } + const char* sSep = jstr(p->pHParam, "flagSep"); + if(sSep == NULL) sSep = jstr(p->pHParam, "flag_sep"); + if(sSep == NULL) sSep = " "; + return DasHttpSrc_outAddFlag(pS, p->sParam, sFVal, sSep, sErr, uErr); + } + + return DasHttpSrc_outSet(pS, p->sParam, bHavePVal ? sPVal : sVal, sErr, uErr); +} + +/* Turn everything the user set (plus required defaults) into parameters. + * Adds to pS->aOut, overriding same named entries, so a 0.6 example's raw + * protocol parameters can be loaded first and still lose to the user's + * keys. */ +static bool DasHttpSrc_build(DasHttpSrc* pS, char* sErr, size_t uErr) +{ + for(size_t u = 0; u < pS->uSet; ++u){ + DasSettable* p = pS->aSet + u; + const char* sVal = NULL; + + if(p->bUserSet) sVal = p->sValue; + else if(p->bRequired && p->bHasDefault) sVal = p->sDefault; + else continue; + + /* A required default fills a hole, it never replaces a value that an + example or -p already supplied */ + if(!p->bUserSet && p->sParam && DasHttpSrc_outFind(pS, p->sParam)) continue; + + /* A value equal to the default carries no information unless the + server insists on hearing it */ + if(p->bUserSet && !p->bRequired && p->bHasDefault){ + bool bSame = false; + if(DasSettable_isBool(p)){ + bool bA = false, bB = false; + bSame = parseBool(sVal, &bA) && parseBool(p->sDefault, &bB) && (bA == bB); + } + else{ + bSame = strieq(sVal, p->sDefault); + } + if(bSame) continue; + } + if(!DasHttpSrc_translate(pS, p, sVal, sErr, uErr)) return false; + } + return true; +} + +/* Every protocol parameter marked required must be present. Servers + * enforce this, and their reply is a 400 whose body we never see, so say it + * here in terms of the interface key the user can set. */ +static bool DasHttpSrc_checkRequired(DasHttpSrc* pS, char* sErr, size_t uErr) +{ + if(!jisdict(pS->pHttpParams)) return true; + for(const das_json_dict_el* pEl = DasJdo_dictFirst(pS->pHttpParams); pEl; pEl = pEl->next){ + if(!jtrue(jget(pEl->value, "required"))) continue; + const char* sName = pEl->name->string; + if(DasHttpSrc_outFind(pS, sName) != NULL) continue; + + size_t u = snprintf(sErr, uErr, "Required parameter missing, set it with"); + int nKeys = 0; + for(size_t i = 0; (i < pS->uSet) && (u < uErr); ++i){ + if(pS->aSet[i].sParam && (strcmp(pS->aSet[i].sParam, sName) == 0)){ + u += snprintf(sErr + u, uErr - u, "%s %s=VALUE", nKeys ? " or" : "", pS->aSet[i].sKey); + ++nKeys; + } + } + if(nKeys == 0) + snprintf(sErr + u, uErr - u, " -p %s=VALUE (no interface key sets it)", sName); + else + snprintf(sErr + u, uErr - u, " (HTTP parameter %s)", sName); + return false; + } + return true; +} + +static size_t DasHttpSrc_baseCount(const DasHttpSrc* pS) +{ + return jisary(pS->pBaseUrls) ? ((const das_json_ary*)pS->pBaseUrls->value)->length : 0; +} + +static const char* DasHttpSrc_baseUrl(const DasHttpSrc* pS, size_t uIdx) +{ + const das_json_ary_el* pEl = DasJdo_aryFirst(pS->pBaseUrls); + for(size_t u = 0; (u < uIdx) && pEl; ++u) pEl = pEl->next; + return pEl ? DasJdo_string(pEl->value) : NULL; +} + +/* Base URL plus the parameters. 0.6 base URLs already carry a query string + * (server=dataset&dataset=...), 0.7 ones do not, hence the joiner check. */ +static bool DasHttpSrc_url(const DasHttpSrc* pS, size_t uBase, char* sUrl, size_t uLen) +{ + const char* sBase = DasHttpSrc_baseUrl(pS, uBase); + if(sBase == NULL) return false; + size_t u = snprintf(sUrl, uLen, "%s", sBase); + const char* sJoin = strchr(sBase, '?') ? "&" : "?"; + char sEnc[VAL_SZ * 3]; + for(size_t i = 0; (i < pS->uOut) && (u < uLen); ++i){ + urlEncode(pS->aOut[i].sVal, sEnc, sizeof(sEnc)); + u += snprintf(sUrl + u, uLen - u, "%s%s=%s", sJoin, pS->aOut[i].sName, sEnc); + sJoin = "&"; + } + return u < uLen; +} + +/* ************************************************************************* */ +/* Examples */ + +/* 0.7 example settings are keyed by interface path, "coords/time/props/min". + * Convert to the dotted key form: drop "props", singular section name. */ +static bool exPathToKey(const char* sPath, char* sKey, size_t uLen) +{ + char sCopy[KEY_SZ * 2]; + strcpyz(sCopy, sPath, sizeof(sCopy)); + size_t u = 0; + int nTok = 0; + sec_e sec; + for(char* pTok = strtok(sCopy, "/"); pTok; pTok = strtok(NULL, "/")){ + if(strcmp(pTok, "props") == 0) continue; + const char* sOut = pTok; + if((nTok == 0) && secAlias(pTok, &sec)) sOut = g_sSecName[sec]; + if(u + strlen(sOut) + 2 >= uLen) return false; + u += snprintf(sKey + u, uLen - u, "%s%s", nTok ? "." : "", sOut); + ++nTok; + } + return nTok > 0; +} + +static size_t DasHttpSrc_exampleCount(const DasHttpSrc* pS) +{ + if(pS->bLegacy){ + const DasJdo* pEx = jget(pS->pProto, "examples"); + return jisdict(pEx) ? ((const das_json_dict*)pEx->value)->length : 0; + } + const DasJdo* pEx = jget(pS->pIface, "examples"); + return jisary(pEx) ? ((const das_json_ary*)pEx->value)->length : 0; +} + +/* The example's own JSON object, or NULL if there is no such example */ +static const DasJdo* DasHttpSrc_example(const DasHttpSrc* pS, size_t uIdx, const char** psName) +{ + if(psName) *psName = NULL; + if(pS->bLegacy){ + const das_json_dict_el* pEl = DasJdo_dictFirst(jget(pS->pProto, "examples")); + for(size_t u = 0; (u < uIdx) && pEl; ++u) pEl = pEl->next; + if(pEl == NULL) return NULL; + if(psName) *psName = pEl->name->string; + return pEl->value; + } + const das_json_ary_el* pEl = DasJdo_aryFirst(jget(pS->pIface, "examples")); + for(size_t u = 0; (u < uIdx) && pEl; ++u) pEl = pEl->next; + return pEl ? pEl->value : NULL; +} + +static const char* DasHttpSrc_exampleLabel(const DasHttpSrc* pS, size_t uIdx) +{ + const char* sName = NULL; + const DasJdo* pEx = DasHttpSrc_example(pS, uIdx, &sName); + if(pEx == NULL) return ""; + const char* sLabel = jstr(pEx, "label"); + if(sLabel == NULL) sLabel = jstr(pEx, "title"); + if(sLabel == NULL) sLabel = sName; + return sLabel ? sLabel : ""; +} + +/* Print the arguments that reproduce an example, indented under its row */ +static void DasHttpSrc_printExampleArgs( + const DasHttpSrc* pS, size_t uIdx, int nCol, FILE* pOut +){ + char sKey[KEY_SZ], sVal[VAL_SZ]; + char sArgs[4096] = {'\0'}; + size_t u = 0; + const DasJdo* pEx = DasHttpSrc_example(pS, uIdx, NULL); + if(pEx == NULL) return; + + for(int i = 0; i < nCol; ++i) fputc(' ', pOut); + if(pS->bLegacy){ + u += snprintf(sArgs + u, sizeof(sArgs) - u, "(protocol parameters) "); + const DasJdo* pParams = jget(pEx, "http_params"); + for(const das_json_dict_el* pP = DasJdo_dictFirst(pParams); pP && (u < sizeof(sArgs)); pP = pP->next){ + if(jscalar(pP->value, sVal, sizeof(sVal))) + u += snprintf(sArgs + u, sizeof(sArgs) - u, "%s=%s ", pP->name->string, sVal); + } + } + else{ + const DasJdo* pSettings = jget(pEx, "settings"); + for(const das_json_dict_el* pP = DasJdo_dictFirst(pSettings); pP && (u < sizeof(sArgs)); pP = pP->next){ + if(exPathToKey(pP->name->string, sKey, sizeof(sKey)) && jscalar(pP->value, sVal, sizeof(sVal))) + u += snprintf(sArgs + u, sizeof(sArgs) - u, "%s=%s ", sKey, sVal); + } + } + fputsWrapped(sArgs, nCol, pOut); + fputs("\n", pOut); +} + +/* The examples table: number and what it provides, arguments only with -v */ +static void DasHttpSrc_printExamples(const DasHttpSrc* pS, FILE* pOut) +{ + size_t uEx = DasHttpSrc_exampleCount(pS); + if(uEx == 0) return; + size_t wKey = 3; + char sNum[16]; + for(size_t u = 0; u < uEx; ++u){ + snprintf(sNum, sizeof(sNum), "%zu", u); + if(strlen(sNum) > wKey) wKey = strlen(sNum); + } + fprintf(pOut, "\nexamples:\n %-*s Provides\n ", (int)wKey, "Key"); + for(size_t i = 0; i < wKey; ++i) fputc('-', pOut); + fputs(" --------\n", pOut); + int nCol = 2 + (int)wKey + 2; + for(size_t u = 0; u < uEx; ++u){ + fprintf(pOut, " %-*zu ", (int)wKey, u); + fputsWrapped(DasHttpSrc_exampleLabel(pS, u), nCol, pOut); + fputc('\n', pOut); + if(g_bVerbose) DasHttpSrc_printExampleArgs(pS, u, nCol, pOut); + } +} + +/* Load example uIdx. 0.7 examples set interface keys and go through the + * normal translation. 0.6 examples are protocol parameters and are applied + * raw after translation, see main(). */ +static bool DasHttpSrc_applyExample( + DasHttpSrc* pS, size_t uIdx, DasHttpParam* aRaw, size_t* puRaw, size_t uMaxRaw, + char* sErr, size_t uErr +){ + if(uIdx >= DasHttpSrc_exampleCount(pS)){ + snprintf(sErr, uErr, "Source has %zu example(s), there is no example %zu", + DasHttpSrc_exampleCount(pS), uIdx); + return false; + } + char sKey[KEY_SZ], sVal[VAL_SZ]; + if(pS->bLegacy){ + const das_json_dict_el* pEl = DasJdo_dictFirst(jget(pS->pProto, "examples")); + for(size_t u = 0; u < uIdx; ++u) pEl = pEl->next; + const DasJdo* pParams = jget(pEl->value, "http_params"); + for(const das_json_dict_el* pP = DasJdo_dictFirst(pParams); pP; pP = pP->next){ + if(!jscalar(pP->value, sVal, sizeof(sVal))) continue; + if(*puRaw >= uMaxRaw){ + snprintf(sErr, uErr, "Example %zu has more than %zu parameters", uIdx, uMaxRaw); + return false; + } + strcpyz(aRaw[*puRaw].sName, pP->name->string, NAME_SZ); + strcpyz(aRaw[*puRaw].sVal, sVal, VAL_SZ); + *puRaw += 1; + } + return true; + } + const das_json_ary_el* pEl = DasJdo_aryFirst(jget(pS->pIface, "examples")); + for(size_t u = 0; u < uIdx; ++u) pEl = pEl->next; + const DasJdo* pSettings = jget(pEl->value, "settings"); + for(const das_json_dict_el* pP = DasJdo_dictFirst(pSettings); pP; pP = pP->next){ + if(!exPathToKey(pP->name->string, sKey, sizeof(sKey))){ + snprintf(sErr, uErr, "Can't interpret example setting path '%s'", pP->name->string); + return false; + } + if(!jscalar(pP->value, sVal, sizeof(sVal))) continue; + if(!DasHttpSrc_set(pS, sKey, sVal, sErr, uErr)) return false; + } + return true; +} + +/* ************************************************************************* */ +/* Stream version and format selection */ + +/* ************************************************************************* */ +/* Describing a source */ + +static void prnAuth(const DasHttpSrc* pS, FILE* pOut) +{ + const DasJdo* pAuth = jget(pS->pProto, "authorization"); + if(pAuth == NULL) pAuth = jget(pS->pProto, "authentication"); + if(pAuth == NULL){ fputs("Auth: unknown\n", pOut); return; } + if(!jtrue(jget(pAuth, "required"))){ fputs("Auth: not required\n", pOut); return; } + fputs("Auth: REQUIRED", pOut); + if(jstr(pAuth, "realm")) fprintf(pOut, ", realm \"%s\"", jstr(pAuth, "realm")); + const DasJdo* pMeth = jget(pAuth, "methods"); + if(jisary(pMeth)){ + fputs(" via", pOut); + for(const das_json_ary_el* pEl = DasJdo_aryFirst(pMeth); pEl; pEl = pEl->next) + fprintf(pOut, " %s", DasJdo_string(pEl->value) ? DasJdo_string(pEl->value) : "?"); + } + if(jstr(pAuth, "description")) fprintf(pOut, "\n %s", jstr(pAuth, "description")); + fputs("\n", pOut); +} + +static void prnFormats(const DasHttpSrc* pS, FILE* pOut) +{ + const DasJdo* pFmts = jget(pS->pIface, "formats"); + if(jisdict(pFmts)){ + fputs("Formats: ", pOut); + for(const das_json_dict_el* pEl = DasJdo_dictFirst(pFmts); pEl; pEl = pEl->next){ + fprintf(pOut, " %s", pEl->name->string); + if(jtrue(jget(pEl->value, "props/enabled/value"))) fputs(" (default)", pOut); + } + fputs("\n", pOut); + return; + } + const char* sMime = jstr(pS->pProto, "format/default/mime"); + if(sMime == NULL) sMime = jstr(pS->pIface, "format/default/mime"); + if(sMime == NULL){ + const DasJdo* pDom = DasNode_getJdo(pS->pNode, NULL); + sMime = jstr(pDom, "format/default/mime"); + } + if(sMime) fprintf(pOut, "Formats: %s\n", sMime); +} + +static void DasHttpSrc_describe(const DasHttpSrc* pS, FILE* pOut, const char* sLabel, const char* sUri) +{ + const DasNode* pNode = pS->pNode; + const char* sVer = DasNode_version(pNode); + fprintf(pOut, "HttpStreamSrc%s%s: %s\n", sVer ? " " : "", sVer ? sVer : "", + DasNode_label(pNode) ? DasNode_label(pNode) : ""); + prnField(pOut, "Title:", DasNode_title(pNode)); + prnField(pOut, "Summary:", DasNode_summary(pNode)); + prnField(pOut, sLabel, sUri); + if(g_bVerbose){ + prnField(pOut, "From:", DasNode_srcUrl(pNode)); + const char* sBase = DasHttpSrc_baseUrl(pS, 0); + if(sBase){ + fprintf(pOut, "Server: %s", sBase); + if(DasHttpSrc_baseCount(pS) > 1) fprintf(pOut, " (+%zu alternate)", DasHttpSrc_baseCount(pS) - 1); + fputs("\n", pOut); + } + } + prnAuth(pS, pOut); + prnFormats(pS, pOut); + + if(g_uShow & SHOW_EXAMPLES) DasHttpSrc_printExamples(pS, pOut); + + if(pS->uSet == 0){ + fputs("\nThis source takes no query parameters.\n", pOut); + return; + } + + if(g_bVerbose) fputs("\nQuery keys, use as KEY=VALUE with 'data' or 'url':\n", pOut); + + /* One table per section, keys shown without their section prefix since + the heading carries it. Column widths are per section so the long + enum lists of the formats section do not push everything else over. */ + enum { C_KEY, C_TYPE, C_UNITS, C_DEF, C_REQ, C_VALS, C_DESC, C_COUNT }; + const char* aHdr[C_COUNT] = {"Key", "Type", "Units", "Default", "Req", "Values", "Description"}; + char sVals[80]; + char sDesc[256]; + + for(int iSec = 0; iSec < SEC_COUNT; ++iSec){ + sec_e sec = (sec_e)iSec; + size_t uInSec = 0; + for(size_t u = 0; u < pS->uSet; ++u) if(pS->aSet[u].sec == sec) ++uInSec; + if(uInSec == 0) continue; + if(!(g_uShow & (1u << sec))) continue; + + size_t aW[C_COUNT]; + for(int c = 0; c < C_COUNT; ++c) aW[c] = strlen(aHdr[c]); + for(size_t u = 0; u < pS->uSet; ++u){ + const DasSettable* p = pS->aSet + u; + if(p->sec != sec) continue; + const char* sShort = strchr(p->sKey, '.') ? strchr(p->sKey, '.') + 1 : p->sKey; + DasSettable_values(p, sVals, sizeof(sVals)); + size_t a[C_COUNT] = { + strlen(sShort), strlen(DasSettable_type(p)), strlen(DasSettable_units(p)), + strlen(p->sDefault), 3, strlen(sVals), 0 + }; + for(int c = 0; c < C_DESC; ++c) if(a[c] > aW[c]) aW[c] = a[c]; + } + int nDescCol = 2; + for(int c = 0; c < C_DESC; ++c) nDescCol += (int)aW[c] + 2; + + fprintf(pOut, "\n%s:\n", g_sSecName[sec]); + fprintf(pOut, " %-*s %-*s %-*s %-*s %-*s %-*s %s\n", + (int)aW[C_KEY], aHdr[C_KEY], (int)aW[C_TYPE], aHdr[C_TYPE], + (int)aW[C_UNITS], aHdr[C_UNITS], (int)aW[C_DEF], aHdr[C_DEF], + (int)aW[C_REQ], aHdr[C_REQ], (int)aW[C_VALS], aHdr[C_VALS], aHdr[C_DESC]); + fputs(" ", pOut); + for(int c = 0; c < C_COUNT; ++c){ + size_t w = (c == C_DESC) ? 24 : aW[c]; + for(size_t i = 0; i < w; ++i) fputc('-', pOut); + if(c < C_DESC) fputs(" ", pOut); + } + fputs("\n", pOut); + + for(size_t u = 0; u < pS->uSet; ++u){ + const DasSettable* p = pS->aSet + u; + if(p->sec != sec) continue; + const char* sShort = strchr(p->sKey, '.') ? strchr(p->sKey, '.') + 1 : p->sKey; + DasSettable_values(p, sVals, sizeof(sVals)); + fprintf(pOut, " %-*s %-*s %-*s %-*s %-*s %-*s ", + (int)aW[C_KEY], sShort, (int)aW[C_TYPE], DasSettable_type(p), + (int)aW[C_UNITS], DasSettable_units(p), (int)aW[C_DEF], p->sDefault, + (int)aW[C_REQ], p->bRequired ? "yes" : "", (int)aW[C_VALS], sVals); + fputsWrapped(DasSettable_desc(p, sDesc, sizeof(sDesc)), nDescCol, pOut); + fputc('\n', pOut); + } + } + /* Usage hints are for the first dozen runs, not the hundredth. They + live in 'data -h' and come back with -v. */ + if(!g_bVerbose) return; + + /* Shortcut hints specific to this source */ + fputs("\nShortcuts:\n", pOut); + char sLastGroup[NAME_SZ] = {'\0'}; + for(size_t u = 0; u < pS->uSet; ++u){ + const DasSettable* p = pS->aSet + u; + if((p->sec != SEC_COORD)||strieq(p->sGroup, sLastGroup)) continue; + strcpyz(sLastGroup, p->sGroup, NAME_SZ); + bool bMin = DasHttpSrc_hasProp(pS, SEC_COORD, p->sGroup, "min"); + bool bMax = DasHttpSrc_hasProp(pS, SEC_COORD, p->sGroup, "max"); + if(!bMin || !bMax) continue; + const char* sStep = DasHttpSrc_hasProp(pS, SEC_COORD, p->sGroup, "res") ? "RES" : + DasHttpSrc_hasProp(pS, SEC_COORD, p->sGroup, "inter") ? "INTER" : NULL; + fprintf(pOut, " %s=MIN,MAX%s%s\n", p->sGroup, sStep ? "," : "", sStep ? sStep : ""); + } + prnPara(pOut, 2, "KEY alone sets a boolean key true, N alone starts from example N, and the " + "section prefix (coords., data., ...) may be dropped when unambiguous. " + "See '" PROG " data -h'."); +} + +/* ************************************************************************* */ +/* Listing a catalog */ + +static void listCatalog(const DasNode* pNode, FILE* pOut, const char* sLabel, const char* sUri) +{ + const char* sType = jstr(DasNode_getJdo(pNode, NULL), D2FRAG_TYPE); + const char* sVer = DasNode_version(pNode); + fprintf(pOut, "%s%s%s: %s\n", sType ? sType : "Node", sVer ? " " : "", sVer ? sVer : "", + DasNode_label(pNode) ? DasNode_label(pNode) : ""); + prnField(pOut, "Title:", DasNode_title(pNode)); + prnField(pOut, sLabel, sUri); + if(g_bVerbose) prnField(pOut, "From:", DasNode_srcUrl(pNode)); + + size_t uKids = DasNode_subCount(pNode); + if(uKids == 0){ fputs("\nNo children listed.\n", pOut); return; } + + size_t wName = 4, wType = 4; + for(size_t u = 0; u < uKids; ++u){ + const char* sName = DasNode_subName(pNode, u); + const char* sT = jstr(DasNode_subStub(pNode, u), D2FRAG_TYPE); + if(sName && (strlen(sName) > wName)) wName = strlen(sName); + if(sT && (strlen(sT) > wType)) wType = strlen(sT); + } + int nDescCol = 2 + (int)wName + 2 + (int)wType + 3; + + fprintf(pOut, "\nChildren (%zu):\n\n", uKids); + bool bUnsupported = false; + for(size_t u = 0; u < uKids; ++u){ + const char* sName = DasNode_subName(pNode, u); + const DasJdo* pStub = DasNode_subStub(pNode, u); + const char* sT = jstr(pStub, D2FRAG_TYPE); + const char* sDesc = jstr(pStub, D2FRAG_TITLE); + if(sDesc == NULL) sDesc = jstr(pStub, D2FRAG_LABEL); + if(sDesc == NULL) sDesc = jstr(pStub, D2FRAG_DESC); + bool bKnown = (DasNode_subType(pNode, u) != d2node_inv); + if(!bKnown) bUnsupported = true; + fprintf(pOut, " %-*s %-*s%s ", (int)wName, sName ? sName : "?", + (int)wType, sT ? sT : "?", bKnown ? " " : "*"); + fputsWrapped(sDesc ? sDesc : "", nDescCol, pOut); + fputc('\n', pOut); + } + if(bUnsupported) + { + fputc('\n', pOut); + prnPara(pOut, 2, "* this node type can be listed but not loaded or fetched by " PROG); + } +} + +/* What to show as a node's name, and the label for it. Under a federated + * root that is the catalog URI (label "URI:"). Under a -r URL or file root + * there is no URI: the root itself shows as "Root:" with the location given, + * its descendants as "Path:" relative to it. Catalogs get their child + * separator appended so the next step is obvious. */ +static const char* nodeUri( + const DasNode* pNode, bool bUrlRoot, const char* sFallback, char* sBuf, size_t uLen +){ + const char* sUri = DasNode_pathUri(pNode); + if((sUri == NULL)||(sUri[0] == '\0')){ + if(bUrlRoot){ + strcpyz(sBuf, sFallback, uLen); + return "Root:"; + } + sUri = sFallback; + } + while((*sUri == '/')||(*sUri == ':')) ++sUri; /* child of an unnamed -r root */ + const char* sSep = DasNode_isCatalog(pNode) ? DasNode_pathSep(pNode) : NULL; + snprintf(sBuf, uLen, "%s%s", sUri, sSep ? sSep : ""); + return bUrlRoot ? "Path:" : "URI:"; +} + +/* ************************************************************************* */ +/* Catalog navigation + * + * DasCat is a pilot for a library object: a catalog session that owns its + * roots, its credentials, its agent and the memory of every node it loads, + * and resolves user paths under one root policy. Today new_RootNode() and + * DasNode_subNode() leave all of that to the caller, which is why this is + * here rather than in node.c; see cf_notes/api_enlargement.md. Only + * applyRootArg(), the command line spelling of the policy, would stay in + * das_fetch when the rest moves. */ + +#define MAX_ROOTS 8 + +typedef struct das_cat { + DasCredMngr* pMgr; /* borrowed, the caller keeps it alive */ + const char* sAgent; /* borrowed */ + + /* Root policy. Relative paths start at sPrefix in the federated + catalog, unless a root URL or a seed is set, then they start at that + node (or, for a graft that also has an explicit prefix, at the prefix). */ + char sPrefix[URI_SZ]; /* default DEF_ROOT */ + bool bPrefixGiven; /* set by the caller, not just the default */ + char sRootUrl[URL_SZ]; /* a node by URL or file as the only root */ + char sSeedUri[URI_SZ]; /* URI=URL: the node at sSeedUrl is named sSeedUri */ + char sSeedUrl[URL_SZ]; + bool bFallThrough; /* the seed patches the federated roots (a graft) */ + + DasNode* pRoot; /* root node of the URL and seed modes, made on first use */ + DasNode* aRoots[MAX_ROOTS]; /* every memory root made, freed by DasCat_free */ + size_t uRoots; + char sLastUri[URI_SZ]; /* the path URI as the last resolve understood it */ +} DasCat; + +static bool isDirectUrl(const char* sPath) +{ + if(sPath == NULL) return false; + if(strpfx(sPath, "http://")||strpfx(sPath, "https://")||strpfx(sPath, "file://")) return true; + if((sPath[0] == '.')||(sPath[0] == '/')||(sPath[0] == '\\')) return true; + size_t u = strlen(sPath); + return (u > 5) && (strcmp(sPath + u - 5, ".json") == 0); +} + +static void DasCat_init(DasCat* pCat, DasCredMngr* pMgr, const char* sAgent) +{ + memset(pCat, 0, sizeof(DasCat)); + pCat->pMgr = pMgr; + pCat->sAgent = sAgent; + strcpyz(pCat->sPrefix, DEF_ROOT, sizeof(pCat->sPrefix)); +} + +/* Frees every node loaded through this catalog. Node pointers handed out + * by DasCat_resolve() are invalid afterwards. */ +static void DasCat_free(DasCat* pCat) +{ + for(size_t u = 0; u < pCat->uRoots; ++u) del_RootNode(pCat->aRoots[u]); + pCat->uRoots = 0; + pCat->pRoot = NULL; +} + +/* Take ownership of a memory root */ +static DasNode* DasCat_keep(DasCat* pCat, DasNode* pRoot) +{ + if(pRoot == NULL) return NULL; + if(pCat->uRoots < MAX_ROOTS) pCat->aRoots[pCat->uRoots++] = pRoot; + else daslog_warn("Too many catalog roots in one run, leaking one"); + return pRoot; +} + +/* site:/x and test:/x are das2py shorthand for the two well known children + * of the das2 root. new_RootNode() expands them too; doing it here as well + * keeps the reported URI the full one. */ +static void DasCat_expand(const char* sIn, char* sOut, size_t uOut) +{ + if(strpfx(sIn, "site:")||strpfx(sIn, "test:")) snprintf(sOut, uOut, D2URI_ROOT "%s", sIn); + else strcpyz(sOut, sIn, uOut); +} + +static bool DasCat_hasRoot(const DasCat* pCat) +{ + return (pCat->sRootUrl[0] != '\0')||(pCat->sSeedUrl[0] != '\0'); +} + +/* The three policy setters return an error text, or NULL */ + +static const char* DasCat_setPrefix(DasCat* pCat, const char* sUri) +{ + if(isDirectUrl(sUri)) return "a path prefix is a catalog URI, not a URL or file"; + DasCat_expand(sUri, pCat->sPrefix, sizeof(pCat->sPrefix)); + pCat->bPrefixGiven = true; + return NULL; +} + +static const char* DasCat_setRoot(DasCat* pCat, const char* sUrl) +{ + if(DasCat_hasRoot(pCat)) return "a catalog has one root"; + if(!isDirectUrl(sUrl)) return "a root is a URL or a file"; + strcpyz(pCat->sRootUrl, sUrl, sizeof(pCat->sRootUrl)); + return NULL; +} + +/* The node at sUrl is named sUri. As the only root (bFallThrough false), + * or patched over the federated catalog (true), see new_RootNode_seed(). */ +static const char* DasCat_seed(DasCat* pCat, const char* sUri, const char* sUrl, bool bFallThrough) +{ + if(DasCat_hasRoot(pCat)) return "a catalog has one root"; + if((sUri == NULL)||(sUri[0] == '\0')) return "a seed needs a URI"; + if(!isDirectUrl(sUrl)) return "a seed needs a URL or file after the '='"; + DasCat_expand(sUri, pCat->sSeedUri, sizeof(pCat->sSeedUri)); + strcpyz(pCat->sSeedUrl, sUrl, sizeof(pCat->sSeedUrl)); + pCat->bFallThrough = bFallThrough; + return NULL; +} + +/* Root node of the URL and seed modes, loaded on first use */ +static DasNode* DasCat_root(DasCat* pCat) +{ + if(pCat->pRoot != NULL) return pCat->pRoot; + if(pCat->sSeedUrl[0] != '\0') + pCat->pRoot = new_RootNode_seed(pCat->sSeedUri, pCat->sSeedUrl, pCat->bFallThrough, + pCat->pMgr, pCat->sAgent); + else if(pCat->sRootUrl[0] != '\0') + pCat->pRoot = new_RootNode_url(pCat->sRootUrl, "", pCat->pMgr, pCat->sAgent); + return DasCat_keep(pCat, pCat->pRoot); +} + +/* Resolve a user path to a node under the root policy. The node is owned + * by the catalog. DasCat_lastUri() afterwards gives the path URI as it was + * understood, resolved or not, for messages. */ +static DasNode* DasCat_resolve(DasCat* pCat, const char* sPath) +{ + char* sUri = pCat->sLastUri; + size_t uLen = sizeof(pCat->sLastUri); + bool bNoPath = (sPath == NULL)||(sPath[0] == '\0'); + + /* A node by URL or file, named by itself */ + if(!bNoPath && isDirectUrl(sPath)){ + strcpyz(sUri, sPath, uLen); + return DasCat_keep(pCat, new_RootNode_url(sPath, sPath, pCat->pMgr, pCat->sAgent)); + } + + if(pCat->sSeedUrl[0] != '\0'){ + DasNode* pRoot = DasCat_root(pCat); + if(pRoot == NULL) return NULL; + + /* No path: the seeded node itself. A full URI, or a relative path + under an explicit prefix when this is a graft: the seed root decides + whether the seed or the federated catalog owns it. Anything else + is relative to the seeded node. */ + if(bNoPath){ + strcpyz(sUri, pCat->sSeedUri, uLen); + return DasNode_subNode(pRoot, pCat->sSeedUri, NULL, NULL); + } + if(pCat->bFallThrough && pCat->bPrefixGiven) + snprintf(sUri, uLen, "%s%s", pCat->sPrefix, sPath); + else + DasCat_expand(sPath, sUri, uLen); + if(strpfx(sUri, "tag:")||strpfx(sUri, pCat->sSeedUri)) + return DasNode_subNode(pRoot, sUri, NULL, NULL); + + DasNode* pSeed = DasNode_subNode(pRoot, pCat->sSeedUri, NULL, NULL); + if(pSeed == NULL){ + strcpyz(sUri, pCat->sSeedUri, uLen); + return NULL; + } + const char* sSep = DasNode_isCatalog(pSeed) ? DasNode_pathSep(pSeed) : "/"; + snprintf(sUri, uLen, "%s%s%s", pCat->sSeedUri, sSep ? sSep : "/", sPath); + return DasNode_subNode(pSeed, sPath, NULL, NULL); + } + + if(pCat->sRootUrl[0] != '\0'){ + DasNode* pRoot = DasCat_root(pCat); + if(pRoot == NULL) return NULL; + if(bNoPath){ + strcpyz(sUri, pCat->sRootUrl, uLen); + return pRoot; + } + strcpyz(sUri, sPath, uLen); + return DasNode_subNode(pRoot, sPath, NULL, NULL); + } + + /* The federated catalog under the prefix */ + if(bNoPath){ + /* The prefix with its trailing separator removed names the container itself */ + strcpyz(sUri, pCat->sPrefix, uLen); + size_t u = strlen(sUri); + while((u > 0) && ((sUri[u-1] == '/')||(sUri[u-1] == ':'))) sUri[--u] = '\0'; + } + else if(strpfx(sPath, "tag:")||strpfx(sPath, "site:")||strpfx(sPath, "test:")) + DasCat_expand(sPath, sUri, uLen); + else + snprintf(sUri, uLen, "%s%s", pCat->sPrefix, sPath); + + return DasCat_keep(pCat, new_RootNode(sUri, pCat->pMgr, pCat->sAgent)); +} + +static const char* DasCat_lastUri(const DasCat* pCat){ return pCat->sLastUri; } + +/* A -r ROOT or -g URI=URL value, or their environment stand-ins, turned into + * DasCat policy. URI=URL seeds the catalog: as the only root for -r, as a + * graft for -g. For -r a URL or file alone is a detached root and anything + * else is a URI prefix; -g accepts only URI=URL. The URL side of a seed + * always has a scheme or looks like a file, which is how the '=' that + * matters is found when the URI itself contains one. Returns an error text + * or NULL. */ +static const char* applyRootArg(DasCat* pCat, const char* sVal, bool bGraft) +{ + const char* sBusy = "only one of -r URL, -r URI=URL or -g URI=URL may be given"; + for(const char* pEq = strchr(sVal, '='); pEq != NULL; pEq = strchr(pEq + 1, '=')){ + if(!isDirectUrl(pEq + 1)) continue; + if(DasCat_hasRoot(pCat)) return sBusy; + if(pEq == sVal){ /* "=URL", a nameless root */ + if(bGraft) return "-g needs URI=URL, a graft without a URI has nowhere to go"; + return DasCat_setRoot(pCat, pEq + 1); + } + char sUri[URI_SZ]; + size_t uLen = (size_t)(pEq - sVal); + if(uLen >= sizeof(sUri)) uLen = sizeof(sUri) - 1; + memcpy(sUri, sVal, uLen); + sUri[uLen] = '\0'; + return DasCat_seed(pCat, sUri, pEq + 1, bGraft); + } + if(bGraft) return "-g needs URI=URL, see -h"; + if(strchr(sVal, '=') != NULL) return "the part after '=' must be a URL or a file"; + if(isDirectUrl(sVal)){ + if(DasCat_hasRoot(pCat)) return sBusy; + return DasCat_setRoot(pCat, sVal); + } + return DasCat_setPrefix(pCat, sVal); +} + +/* From a node the user named, get the HttpStreamSrc to query. Containers + * of sources pick a child: 'flex', else the first stream source listed. To + * pick another the user puts its name on the end of the path. */ +static DasNode* pickSource(DasNode* pNode, char* sErr, size_t uErr) +{ + if(DasNode_isStreamSrc(pNode)) return pNode; + + if(!DasNode_isCatalog(pNode)){ + snprintf(sErr, uErr, "Node %s is not a data source", DasNode_srcUrl(pNode)); + return NULL; + } + + size_t uKids = DasNode_subCount(pNode); + const char* sPick = NULL; + bool bHasCatalogs = false; + + for(size_t u = 0; u < uKids; ++u){ + const char* sName = DasNode_subName(pNode, u); + das_node_type_e nType = DasNode_subType(pNode, u); + if((nType == d2node_catalog)||(nType == d2node_spdf_cat)||(nType == d2node_spase_cat)) + bHasCatalogs = true; + if(nType != d2node_stream_src) continue; + if(strieq(sName, "flex")){ sPick = sName; break; } + if(sPick == NULL) sPick = sName; + } + + if(sPick == NULL){ + if(bHasCatalogs) + snprintf(sErr, uErr, "This is a catalog, not a data source. Run '" PROG + " node' on it and pick a child"); + else + snprintf(sErr, uErr, "No HttpStreamSrc children under %s", DasNode_srcUrl(pNode)); + return NULL; + } + + /* Credentials and agent are inherited from the node's root */ + DasNode* pSub = DasNode_subNode(pNode, sPick, NULL, NULL); + if(pSub == NULL){ + snprintf(sErr, uErr, "Couldn't load source '%s' from %s", sPick, DasNode_srcUrl(pNode)); + return NULL; + } + if(!DasNode_isStreamSrc(pSub)){ + snprintf(sErr, uErr, "Child '%s' loaded but is not an HttpStreamSrc", sPick); + return NULL; + } + return pSub; +} + +/* A path did not resolve. Resolve its parent and say which child was the + * problem and what the parent offers. Call right after the failed + * DasCat_resolve(), while DasCat_lastUri() still describes it. */ +static void explainUnresolved(DasCat* pCat, const char* sPath) +{ + char sFullUri[URI_SZ]; + strcpyz(sFullUri, DasCat_lastUri(pCat), sizeof(sFullUri)); + + /* The common mistake with a restricted root is asking for something it + does not cover, say so instead of hunting for a parent */ + if((pCat->sSeedUrl[0] != '\0') && !pCat->bFallThrough && !strpfx(sFullUri, pCat->sSeedUri)){ + daslog_error_v("'%s' is outside the -r root %s, use -g to patch the " + "federated catalog instead", sFullUri, pCat->sSeedUri); + return; + } + + char sParent[URI_SZ]; + strcpyz(sParent, sPath ? sPath : "", sizeof(sParent)); + size_t u = strlen(sParent); + while((u > 0) && ((sParent[u-1] == '/')||(sParent[u-1] == ':'))) sParent[--u] = '\0'; + char* pLast = strrchr(sParent, '/'); + if((pLast == NULL)||(pLast == sParent)){ + daslog_error_v("Couldn't resolve catalog path '%s'", sFullUri); + return; + } + *pLast = '\0'; + const char* sChild = pLast + 1; + + DasNode* pNode = DasCat_resolve(pCat, sParent); + const char* sParentUri = DasCat_lastUri(pCat); + if((pNode == NULL)||!DasNode_isCatalog(pNode)){ + daslog_error_v("Couldn't resolve catalog path '%s'", sFullUri); + return; + } + + char sMsg[ERR_SZ]; + size_t uKids = DasNode_subCount(pNode); + size_t n = 0; + bool bFound = false; + for(u = 0; u < uKids; ++u){ + const char* sName = DasNode_subName(pNode, u); + if(strieq(sName, sChild)){ + const char* sType = jstr(DasNode_subStub(pNode, u), D2FRAG_TYPE); + if(DasNode_subType(pNode, u) == d2node_inv) + n = snprintf(sMsg, sizeof(sMsg), "'%s' under %s is a %s node, which " PROG + " can list but not load or fetch from", sChild, sParentUri, sType ? sType : "?"); + else + n = snprintf(sMsg, sizeof(sMsg), "'%s' under %s is listed but could not be loaded", + sChild, sParentUri); + bFound = true; + break; + } + } + if(!bFound) n = snprintf(sMsg, sizeof(sMsg), "No child named '%s' under %s", sChild, sParentUri); + if(uKids > 0){ + n += snprintf(sMsg + n, sizeof(sMsg) - n, ". Children:"); + for(u = 0; (u < uKids) && (n < sizeof(sMsg)); ++u) + n += snprintf(sMsg + n, sizeof(sMsg) - n, " %s", DasNode_subName(pNode, u)); + } + daslog_error_v("%s", sMsg); +} + +/* Everything main() owns, in dependency order: nodes borrow the credential + * manager, so they go first. Returns its argument so exits read as one line. */ +static int finish(DasCat* pCat, int nRet) +{ + DasCat_free(pCat); + if(pCat->pMgr != NULL) del_CredMngr(pCat->pMgr); + pCat->pMgr = NULL; + return nRet; +} + +/* ************************************************************************* */ +/* Fetching */ + +static int streamUrl(DasCat* pCat, const char* sUrl, float rTimeout) +{ + DasHttpResp res; + memset(&res, 0, sizeof(DasHttpResp)); + + daslog_info_v("Requesting %s", sUrl); + + if(!das_http_getBody(sUrl, pCat->sAgent, pCat->pMgr, &res, rTimeout)){ + int nRet = RET_SERVER; + if((res.nCode == 401)||(res.nCode == 403)) nRet = RET_AUTH; + else if(res.nCode == 400) nRet = RET_QUERY; + daslog_error_v("%s", res.sError ? res.sError : "Request failed"); + if(res.nCode == 400) + daslog_error("The server rejected the query. Check the keys with '" + PROG " node' or try the URL from '" PROG " url' in a browser"); + DasHttpResp_freeFields(&res); + return nRet; + } + + char sFinal[URL_SZ]; + das_url_toStr(&(res.url), sFinal, sizeof(sFinal)); + if(strcmp(sFinal, sUrl) != 0) daslog_info_v("Redirected to %s", sFinal); + if(res.pMime) daslog_info_v("Content type %s", res.pMime); + + das_stdio_binary(stdout); + + DasIO* pIn = DasHttpResp_useSsl(&res) ? + new_DasIO_ssl(PROG, res.pSsl, "r") : new_DasIO_socket(PROG, res.nSockFd, "r"); + DasBuf* pBuf = new_DasBuf(CHUNK_SZ); + + int nRet = RET_OK; + int64_t nTotal = 0; + while(true){ + DasBuf_reinit(pBuf); + int nRead = DasIO_read(pIn, pBuf, CHUNK_SZ); + if(nRead < 0){ nRet = RET_SERVER; break; } + if(nRead == 0) break; + size_t uLen = 0; + const ubyte* pBytes = DasBuf_direct(pBuf, &uLen); + if(fwrite(pBytes, 1, uLen, stdout) != uLen){ + daslog_error_v("Write to standard output failed, %s", strerror(errno)); + nRet = RET_WRITE; + break; + } + nTotal += nRead; + } + fflush(stdout); + + DasIO_close(pIn); + del_DasIO(pIn); + del_DasBuf(pBuf); + DasHttpResp_freeFields(&res); + + daslog_info_v("%" PRId64 " bytes written", nTotal); + return nRet; +} + +/* ************************************************************************* */ + +static bool isDigits(const char* s) +{ + if((s == NULL)||(*s == '\0')) return false; + for(; *s; ++s) if(!isdigit((unsigned char)*s)) return false; + return true; +} + +/* Columns available on stdout, 0 when it is not a terminal */ +static int termWidth(void) +{ + if(!isatty(fileno(stdout))) return 0; +#ifdef _WIN32 + CONSOLE_SCREEN_BUFFER_INFO info; + if(GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &info)) + return info.srWindow.Right - info.srWindow.Left + 1; +#else + struct winsize ws; + if((ioctl(fileno(stdout), TIOCGWINSZ, &ws) == 0) && (ws.ws_col > 0)) return ws.ws_col; +#endif + const char* sCols = getenv("COLUMNS"); + if(isDigits(sCols)) return atoi(sCols); + return 80; +} + +/* -s SECTIONS: comma separated section names to a bit set */ +static bool parseShow(const char* sList, unsigned* puShow) +{ + char sCopy[256]; + strcpyz(sCopy, sList, sizeof(sCopy)); + *puShow = 0; + for(char* pTok = strtok(sCopy, ","); pTok != NULL; pTok = strtok(NULL, ",")){ + sec_e sec; + if(strieq(pTok, "all")) *puShow = SHOW_ALL; + else if(strieq(pTok, "examples")||strieq(pTok, "example")||strieq(pTok, "ex")) *puShow |= SHOW_EXAMPLES; + else if(secAlias(pTok, &sec)) *puShow |= (1u << sec); + else return false; + } + return true; +} + +/* The default cache directory. Everything das keeps for a user lives under + * one dot directory in the home directory on every OS, the way ssh, aws and + * friends do it, so the documentation names one place instead of three. */ +static void defCacheDir(char* sBuf, size_t uLen) +{ + sBuf[0] = '\0'; + const char* sHome = das_userhome(); + if(sHome && sHome[0]) + snprintf(sBuf, uLen, "%s" DAS_DSEPS DAS_DIR DAS_DSEPS "cache" DAS_DSEPS "nodes", sHome); +} + +int main(int argc, char** argv) +{ + das_init(PROG, DASERR_DIS_EXIT, 0, DASLOG_WARN, NULL); + daslog_set_showline(DASLOG_NOTHING); /* messages here are for users, not debuggers */ + + char sAgent[64]; + snprintf(sAgent, sizeof(sAgent), PROG " (das2C %s)", das_version()); + DasCat cat; + DasCat_init(&cat, NULL, sAgent); /* credentials attach once the manager exists */ + + const char* sCmd = NULL; + const char* sPath = NULL; + const char* sCacheArg = NULL; + const char* sRootArg = NULL; + const char* sGraftArg = NULL; + bool bHelp = false; + bool bFresh = false; + bool bQuiet = false; + const char* aKeyVals[MAX_SETTABLES]; + size_t uKeyVals = 0; + DasHttpParam aRaw[MAX_RAW]; + size_t uRaw = 0; + int nExample = -1; + float rTimeout = 10.0; + char sErr[ERR_SZ] = {'\0'}; + + for(int i = 1; i < argc; ++i){ + const char* sArg = argv[i]; + if((strcmp(sArg, "-h") == 0)||(strcmp(sArg, "--help") == 0)){ bHelp = true; continue; } + if((strcmp(sArg, "-V") == 0)||(strcmp(sArg, "--version") == 0)){ + printf(PROG " (das2C %s)\n", das_version()); + return RET_OK; + } + if(strcmp(sArg, "-q") == 0){ daslog_setlevel(DASLOG_ERROR); bQuiet = true; continue; } + if(strcmp(sArg, "-v") == 0){ daslog_setlevel(DASLOG_INFO); g_bVerbose = true; g_uShow = SHOW_ALL; continue; } + if(strcmp(sArg, "-f") == 0){ bFresh = true; continue; } + if(strcmp(sArg, "--root") == 0) sArg = "-r"; + if(strcmp(sArg, "--graft") == 0) sArg = "-g"; + if((strcmp(sArg, "-r") == 0)||(strcmp(sArg, "-g") == 0)||(strcmp(sArg, "-s") == 0)|| + (strcmp(sArg, "-C") == 0)||(strcmp(sArg, "-p") == 0)||(strcmp(sArg, "-t") == 0)|| + (strcmp(sArg, "-l") == 0)){ + if(i + 1 >= argc){ + fprintf(stderr, PROG ": option %s needs a value, use -h for help\n", sArg); + return RET_USAGE; + } + const char* sVal = argv[++i]; + switch(sArg[1]){ + case 'r': sRootArg = sVal; break; + case 'g': sGraftArg = sVal; break; + case 's': + if(!parseShow(sVal, &g_uShow)){ + fprintf(stderr, PROG ": -s takes a list from coords, data, formats, options, " + "examples, all\n"); + return RET_USAGE; + } + break; + case 'C': sCacheArg = sVal; break; + case 't': rTimeout = (float)atof(sVal); break; + case 'l':{ + int nLvl = daslog_strlevel(sVal); + daslog_setlevel(nLvl); + break; + } + case 'p':{ + const char* pEq = strchr(sVal, '='); + if((pEq == NULL)||(pEq == sVal)||(uRaw >= MAX_RAW)){ + fprintf(stderr, PROG ": -p needs PARAM=VALUE (at most %d of them)\n", MAX_RAW); + return RET_USAGE; + } + size_t uName = (size_t)(pEq - sVal); + if(uName >= NAME_SZ) uName = NAME_SZ - 1; + memcpy(aRaw[uRaw].sName, sVal, uName); + aRaw[uRaw].sName[uName] = '\0'; + strcpyz(aRaw[uRaw].sVal, pEq + 1, VAL_SZ); + ++uRaw; + break; + } + } + continue; + } + if(sArg[0] == '-'){ + fprintf(stderr, PROG ": unknown option %s, use -h for help\n", sArg); + return RET_USAGE; + } + + /* Positionals: command, path, then KEY=VALUE pairs. A bare number is + an example index, not a path. */ + if(sCmd == NULL){ sCmd = sArg; continue; } + if((sPath == NULL) && (strchr(sArg, '=') == NULL) && !isDigits(sArg)){ sPath = sArg; continue; } + if(uKeyVals < MAX_SETTABLES) aKeyVals[uKeyVals++] = sArg; + } + if(bHelp){ prnHelp(sCmd); return RET_OK; } + g_nCols = termWidth(); + + /* Environment fallbacks for -r and -g, for people who live in one test + catalog. The root is applied first so a graft can complain about a + conflicting one. */ + const char* sEnv = getenv(ROOT_ENV); + if((sRootArg == NULL) && (sEnv != NULL) && (sEnv[0] != '\0')) sRootArg = sEnv; + sEnv = getenv(GRAFT_ENV); + if((sGraftArg == NULL) && (sEnv != NULL) && (sEnv[0] != '\0')) sGraftArg = sEnv; + const char* sBad = NULL; + if((sRootArg != NULL) && ((sBad = applyRootArg(&cat, sRootArg, false)) != NULL)){ + fprintf(stderr, PROG ": %s\n", sBad); + return RET_USAGE; + } + if((sGraftArg != NULL) && ((sBad = applyRootArg(&cat, sGraftArg, true)) != NULL)){ + fprintf(stderr, PROG ": %s\n", sBad); + return RET_USAGE; + } + + if(sCmd == NULL){ + fprintf(stderr, PROG ": no command given, use -h for help\n"); + return RET_USAGE; + } + + if(strcmp(sCmd, "roots") == 0){ + size_t uRoots = 0; + const char** psRoots = das_root_urls(&uRoots); + for(size_t u = 0; u < uRoots; ++u) printf("%s\n", psRoots[u]); + return RET_OK; + } + + bool bNode = (strcmp(sCmd, "node") == 0)||(strcmp(sCmd, "ls") == 0); + bool bJson = (strcmp(sCmd, "json") == 0); + bool bUrl = (strcmp(sCmd, "url") == 0); + bool bData = (strcmp(sCmd, "data") == 0)||(strcmp(sCmd, "get") == 0); + if(!bNode && !bJson && !bUrl && !bData){ + fprintf(stderr, PROG ": unknown command '%s', use -h for help\n", sCmd); + return RET_USAGE; + } + if((bJson || bUrl || bData) && (sPath == NULL) && !DasCat_hasRoot(&cat)){ + fprintf(stderr, PROG ": the %s command needs a PATH, use -h for help\n", sCmd); + return RET_USAGE; + } + if(bNode && (uKeyVals > 0)) + daslog_warn("KEY=VALUE arguments are ignored by the node command"); + + /* Node cache, unless told otherwise. -f refreshes rather than disables + so that the run after a catalog change leaves the cache current. */ + char sCacheDir[512] = {'\0'}; + if(sCacheArg == NULL) defCacheDir(sCacheDir, sizeof(sCacheDir)); + else if(!strieq(sCacheArg, "none")) strcpyz(sCacheDir, sCacheArg, sizeof(sCacheDir)); + if(sCacheDir[0] != '\0') DasNode_setCache(sCacheDir, bFresh ? 0 : DEF_CACHE_AGE); + + /* Credentials: saved ones are tried before the terminal prompt */ + char sCredFile[256] = {'\0'}; + const char* sHome = das_userhome(); + if(sHome && sHome[0]) + snprintf(sCredFile, sizeof(sCredFile), "%s" DAS_DSEPS DAS_DIR DAS_DSEPS CRED_FILE, sHome); + cat.pMgr = new_CredMngr(sCredFile[0] ? sCredFile : NULL); + if(sCredFile[0]){ + /* CredMngr_load logs an error for a missing file even though its + contract says a missing file is not an error. Most users have no + saved credentials, so look first. */ + FILE* pTest = fopen(sCredFile, "r"); + if(pTest != NULL){ + fclose(pTest); + CredMngr_load(cat.pMgr, NULL, NULL); + } + } + + /* A fragment on the path is only meaningful for the json command */ + char sPathBuf[1024] = {'\0'}; + const char* sFrag = NULL; + if(sPath != NULL){ + strcpyz(sPathBuf, sPath, sizeof(sPathBuf)); + char* pHash = strchr(sPathBuf, '#'); + if(pHash != NULL){ *pHash = '\0'; sFrag = pHash + 1; } + /* "a/b/" means the same as "a/b", shell completion adds the slash */ + if(!isDirectUrl(sPathBuf)){ + size_t u = strlen(sPathBuf); + while((u > 1) && (sPathBuf[u-1] == '/') && (sPathBuf[u-2] != ':')) sPathBuf[--u] = '\0'; + } + sPath = sPathBuf; + } + + DasNode* pNode = DasCat_resolve(&cat, sPath); + if(pNode == NULL){ + explainUnresolved(&cat, sPath); + return finish(&cat, RET_NODE); + } + const char* sFullUri = DasCat_lastUri(&cat); + char sShow[1200]; + bool bNoUri = (cat.sRootUrl[0] != '\0') || isDirectUrl(sPath); /* URL or file, not a catalog URI */ + const char* sLabel = nodeUri(pNode, bNoUri, sFullUri, sShow, sizeof(sShow)); + + int nRet = RET_OK; + + if(bJson){ + const DasJdo* pJdo = DasNode_getJdo(pNode, sFrag); + if(pJdo == NULL){ + daslog_error_v("Node has no element at '%s'", sFrag ? sFrag : ""); + nRet = RET_NODE; + } + else{ + char* sPretty = (char*)DasJdo_writePretty(pJdo, " ", "\n", NULL); + if(sPretty){ fputs(sPretty, stdout); fputs("\n", stdout); free(sPretty); } + } + return finish(&cat, nRet); + } + + if(bNode){ + if(DasNode_isCatalog(pNode)){ + /* A source container is one step from the goal. Users never type + the source's name, so by default act as if they had asked for the + default source and describe it under the container's URI. The + container listing itself only matters once more than one source + type can be fetched from, so it is verbose only for now. */ + das_node_type_e nType = DasNode_type(pNode); + bool bSrcSet = (nType == d2node_srcset)||(nType == d2node_collection); + if(!bSrcSet || g_bVerbose) listCatalog(pNode, stdout, sLabel, sShow); + + if(bSrcSet){ + DasNode* pSrc = pickSource(pNode, sErr, sizeof(sErr)); + if(pSrc == NULL){ + daslog_error_v("%s", sErr); + nRet = RET_NODE; + } + else{ + DasHttpSrc src; + if(!DasHttpSrc_init(&src, pSrc, sErr, sizeof(sErr))){ + daslog_error_v("%s", sErr); + nRet = RET_NODE; + } + else if(!g_bVerbose){ + DasHttpSrc_describe(&src, stdout, sLabel, sShow); + } + else{ + /* The child's name is the tail of its path URI, after the + separator this container uses */ + const char* sSrcUri = DasNode_pathUri(pSrc); + const char* sSep = DasNode_pathSep(pNode); + const char* sName = sSrcUri; + if(sSrcUri && sSep && sSep[0]){ + const char* p = strstr(sSrcUri, sSep); + while(p){ sName = p + strlen(sSep); p = strstr(sName, sSep); } + } + char sNote[256]; + snprintf(sNote, sizeof(sNote), "Default source: %s", sName ? sName : "first"); + fputc('\n', stdout); + prnPara(stdout, 0, sNote); + fputc('\n', stdout); + char sSrcShow[1200]; + const char* sSrcLabel = nodeUri(pSrc, bNoUri, + sSrcUri ? sSrcUri : "", sSrcShow, sizeof(sSrcShow)); + DasHttpSrc_describe(&src, stdout, sSrcLabel, sSrcShow); + } + } + } + } + else if(DasNode_isStreamSrc(pNode)){ + DasHttpSrc src; + if(!DasHttpSrc_init(&src, pNode, sErr, sizeof(sErr))){ + daslog_error_v("%s", sErr); + nRet = RET_NODE; + } + else + DasHttpSrc_describe(&src, stdout, sLabel, sShow); + } + else{ + const char* sType = jstr(DasNode_getJdo(pNode, NULL), D2FRAG_TYPE); + printf("%s: %s\n%-10s%s\n", sType ? sType : "Node", + DasNode_label(pNode) ? DasNode_label(pNode) : "", sLabel, sShow); + if(g_bVerbose) printf("From: %s\n", DasNode_srcUrl(pNode)); + printf("\n" PROG " has no detailed printer for this node type, try the json command.\n"); + } + return finish(&cat, nRet); + } + + /* url or data */ + DasNode* pSrc = pickSource(pNode, sErr, sizeof(sErr)); + if(pSrc == NULL){ + daslog_error_v("%s", sErr); + return finish(&cat, RET_NODE); + } + + DasHttpSrc src; + if(!DasHttpSrc_init(&src, pSrc, sErr, sizeof(sErr))){ + daslog_error_v("%s", sErr); + return finish(&cat, RET_NODE); + } + + /* Order matters: example first, then explicit keys override it, and -p + raw parameters have the last word. A 0.6 example is raw protocol + parameters, those go straight into the output list before the keys + are translated on top of them. */ + DasHttpParam aExRaw[MAX_RAW]; + size_t uExRaw = 0; + + /* Example selectors among the keys: examples.N, or bare N as the shortcut, + the same pairing as data.sprt and sprt. They are pulled out first since + an example is the base the other keys modify. */ + size_t uKept = 0; + for(size_t u = 0; u < uKeyVals; ++u){ + const char* sArg = aKeyVals[u]; + const char* sNum = sArg; + if(strchr(sArg, '=') == NULL){ + const char* pDot = strchr(sArg, '.'); + if(pDot != NULL){ + size_t uLen = (size_t)(pDot - sArg); + bool bSec = ((uLen == 7) && (strncmp(sArg, "example", 7) == 0)) || + ((uLen == 8) && (strncmp(sArg, "examples", 8) == 0)); + if(bSec){ + if(!isDigits(pDot + 1)){ + daslog_error_v("Examples are numbered, there is no '%s'", sArg); + return finish(&cat, RET_QUERY); + } + sNum = pDot + 1; + } + } + if(isDigits(sNum)){ nExample = atoi(sNum); continue; } + } + aKeyVals[uKept++] = sArg; + } + uKeyVals = uKept; + + /* Nothing asked for at all: run the first example so there is always a + quick way to see a source work, and say so */ + if((nExample < 0) && (uKeyVals == 0) && (uRaw == 0) && (DasHttpSrc_exampleCount(&src) > 0)){ + nExample = 0; + if(!bQuiet) + fprintf(stderr, PROG ": no query given, using example 0, \"%s\"\n", DasHttpSrc_exampleLabel(&src, 0)); + } + + if(nExample >= 0){ + if(!DasHttpSrc_applyExample(&src, (size_t)nExample, aExRaw, &uExRaw, MAX_RAW, sErr, sizeof(sErr))){ + daslog_error_v("%s", sErr); + return finish(&cat, RET_QUERY); + } + } + for(size_t u = 0; u < uExRaw; ++u){ + if(!DasHttpSrc_outSet(&src, aExRaw[u].sName, aExRaw[u].sVal, sErr, sizeof(sErr))){ + daslog_error_v("%s", sErr); + return finish(&cat, RET_QUERY); + } + } + for(size_t u = 0; u < uKeyVals; ++u){ + char sKey[KEY_SZ]; + const char* pEq = strchr(aKeyVals[u], '='); + const char* sVal = "true"; /* bare KEY toggles a boolean on */ + if(pEq != NULL){ + size_t uLen = (size_t)(pEq - aKeyVals[u]); + if(uLen >= KEY_SZ) uLen = KEY_SZ - 1; + memcpy(sKey, aKeyVals[u], uLen); + sKey[uLen] = '\0'; + sVal = pEq + 1; + } + else{ + strcpyz(sKey, aKeyVals[u], KEY_SZ); + } + if(!DasHttpSrc_set(&src, sKey, sVal, sErr, sizeof(sErr))){ + daslog_error_v("%s", sErr); + return finish(&cat, RET_QUERY); + } + } + + if(!DasHttpSrc_build(&src, sErr, sizeof(sErr))){ + daslog_error_v("%s", sErr); + return finish(&cat, RET_QUERY); + } + for(size_t u = 0; u < uRaw; ++u){ + if(!DasHttpSrc_outSet(&src, aRaw[u].sName, aRaw[u].sVal, sErr, sizeof(sErr))){ + daslog_error_v("%s", sErr); + return finish(&cat, RET_QUERY); + } + } + if(!DasHttpSrc_checkRequired(&src, sErr, sizeof(sErr))){ + daslog_error_v("%s", sErr); + return finish(&cat, RET_QUERY); + } + + char sUrl[URL_SZ]; + if(bUrl){ + if(!DasHttpSrc_url(&src, 0, sUrl, sizeof(sUrl))){ + daslog_error("Query URL too long or no base URL"); + nRet = RET_QUERY; + } + else{ + printf("%s\n", sUrl); + } + return finish(&cat, nRet); + } + + /* data: try each base URL in turn until one answers. A server that + answers with an error is an answer, only connection failures move on */ + size_t uBases = DasHttpSrc_baseCount(&src); + nRet = RET_SERVER; + for(size_t u = 0; u < uBases; ++u){ + if(!DasHttpSrc_url(&src, u, sUrl, sizeof(sUrl))){ + daslog_error("Query URL too long"); + nRet = RET_QUERY; + break; + } + nRet = streamUrl(&cat, sUrl, rTimeout); + if((nRet != RET_SERVER)||(u + 1 >= uBases)) break; + daslog_warn_v("Trying alternate server %s", DasHttpSrc_baseUrl(&src, u + 1)); + } + + finish(&cat, nRet); + das_finish(); + return nRet; +} From 2a1c4aa7d69c47d340cb39727c40507e7d2da8b6 Mon Sep 17 00:00:00 2001 From: Chris Piker Date: Sun, 25 Oct 2026 03:28:02 -0500 Subject: [PATCH 02/18] http: streams a response body to a FILE * New function das_http_copyBody, a counter part for das_http_readUrl bug fix: New close helper frees the SSL context bug fix: always-true limit warning in das_http_readUrl Developed-with: Claude Fable 5.1 --- das3/http.c | 158 +++++++++++++++++++++++++++-------------- das3/http.h | 44 +++++++++++- utilities/das_fetch.c | 32 +++------ utilities/validrange.h | 2 - 4 files changed, 155 insertions(+), 81 deletions(-) diff --git a/das3/http.c b/das3/http.c index 94e52c2..b35e44c 100644 --- a/das3/http.c +++ b/das3/http.c @@ -1,6 +1,6 @@ -/* Copyright (C) 2017-2019 Chris Piker +/* Copyright (C) 2017-2026 Chris Piker * - * This file is part of libdas2, the Core Das2 C Library. + * This file is part of das2C, the Core Das2 C Library. * * Libdas2 is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License version 2.1 as published @@ -12,7 +12,7 @@ * more details. * * You should have received a copy of the GNU Lesser General Public License - * version 2.1 along with libdas2; if not, see . + * version 2.1 along with das2C; if not, see . */ #define _POSIX_C_SOURCE 200112L @@ -1205,10 +1205,59 @@ bool das_http_getBody( /* ************************************************************************* */ -/* A just-give-me-a-bag-of-bytes convenience function */ +/* Body readers: one chunk at a time, shared by the slurp and copy functions */ #define D2CHAR_CHUNK_SZ 16384 +/* Returns bytes read, 0 at end of body, negative on a transport error. + * Errors are logged here so both public callers report them the same way. */ +static int _das_http_readChunk(DasHttpResp* pRes, char* pBuf, size_t uLen) +{ + int nRead = 0; + if(pRes->pSsl != NULL){ + nRead = SSL_read((SSL*)pRes->pSsl, pBuf, (int)uLen); + if(nRead < 0){ + char* sErr = das_ssl_getErr(pRes->pSsl, nRead); + daslog_error_v("Error reading from SSL socket, %s", sErr); + free(sErr); + } + } + else{ + errno = 0; + nRead = recv(pRes->nSockFd, pBuf, uLen, 0); + if(nRead < 0) + daslog_error_v("Error reading from socket, %s", strerror(errno)); + } + return nRead; +} + +/* Release the transport. das_http_getBody() creates the SSL object, so it + * is freed here and not by DasIO_close(), which only knows the socket. */ +static void _das_http_closeBody(DasHttpResp* pRes) +{ + if(pRes->pSsl != NULL){ + pRes->nSockFd = SSL_get_fd((SSL*)pRes->pSsl); + if(SSL_shutdown((SSL*)pRes->pSsl) == 0) /* 0 means two-stage shutdown */ + SSL_shutdown((SSL*)pRes->pSsl); + SSL_free((SSL*)pRes->pSsl); + pRes->pSsl = NULL; + } + if(pRes->nSockFd < 0) return; + + daslog_debug_v("Shutting down socket %d", pRes->nSockFd); +#ifndef _WIN32 + shutdown(pRes->nSockFd, SHUT_RDWR); + close(pRes->nSockFd); +#else + shutdown(pRes->nSockFd, SD_BOTH); + closesocket(pRes->nSockFd); +#endif + pRes->nSockFd = -1; +} + +/* ************************************************************************* */ +/* A just-give-me-a-bag-of-bytes convenience function */ + DasAry* das_http_readUrl( const char* sUrl, const char* sAgent, DasCredMngr* pMgr, DasHttpResp* pRes, int64_t nLimit, float rConSec @@ -1221,68 +1270,69 @@ DasAry* das_http_readUrl( DasAry* pAry = new_DasAry("http_body", vtUByte, 1, NULL, RANK_1(0), UNIT_DIMENSIONLESS); - SSL* pSsl = (SSL*)pRes->pSsl; char buf[D2CHAR_CHUNK_SZ]; int nRead = 0; - if(DasHttpResp_useSsl(pRes)){ - while((nLimit == -1)||(nTotal < nLimit)){ - nRead = SSL_read(pSsl, buf, D2CHAR_CHUNK_SZ); - - if(nRead == 0) break; /* Socket is done */ - - if(nRead < 1){ /* Socket is broke */ - char* sErr = das_ssl_getErr(pSsl, nRead); - daslog_error_v("Error reading from SSL socket, %s", sErr); - free(sErr); - dec_DasAry(pAry); - return false; - } - - nTotal += nRead; - if(!DasAry_append(pAry, (const ubyte*) buf, nRead)) return false; /* Yay data! */ + while((nLimit == -1)||(nTotal < nLimit)){ + nRead = _das_http_readChunk(pRes, buf, D2CHAR_CHUNK_SZ); + if(nRead == 0) break; /* Socket is done */ + if(nRead < 0){ /* Socket is broke */ + dec_DasAry(pAry); + pAry = NULL; + break; + } + nTotal += nRead; + if(!DasAry_append(pAry, (const ubyte*) buf, nRead)){ + dec_DasAry(pAry); + pAry = NULL; + break; } - - pRes->nSockFd = SSL_get_fd((SSL*)pRes->pSsl); - if(SSL_shutdown((SSL*)pRes->pSsl) == 0) // If 0 needs 2-stage shutdown - SSL_shutdown((SSL*)pRes->pSsl); - - SSL_free((SSL*)pRes->pSsl); - pRes->pSsl = NULL; } - else{ - while((nLimit == -1)||(nTotal < nLimit)){ - errno = 0; - nRead = recv(pRes->nSockFd, buf, D2CHAR_CHUNK_SZ, 0); - if(nRead == 0) break; /* Socket is done */ + _das_http_closeBody(pRes); + if(pAry == NULL) return NULL; - if(nRead < 0){ /* Socket is broke */ - daslog_error_v("Error reading from socket, %s", strerror(errno)); - dec_DasAry(pAry); - return false; - } - - nTotal += nRead; - if(!DasAry_append(pAry, (const ubyte*) buf, nRead)) return false; /* Yay data! */ - } - } - if(nTotal > nLimit){ + if((nLimit > 0)&&(nTotal > nLimit)){ daslog_warn_v("Limit of %" PRId64 " bytes hit, almost certainly returning a " "partial download", nLimit); } else daslog_debug_v("%ld bytes read from %s", DasAry_size(pAry), sUrl); - daslog_debug_v("Shutting down socket %d", pRes->nSockFd); -#ifndef _WIN32 - shutdown(pRes->nSockFd, SHUT_RDWR); - close(pRes->nSockFd); -#else - shutdown(pRes->nSockFd, SD_BOTH); - closesocket(pRes->nSockFd); -#endif - pRes->nSockFd = -1; - return pAry; } + +/* ************************************************************************* */ +/* Streaming counterpart, for bodies too big or too slow to hold in memory */ + +DasErrCode das_http_copyBody( + DasHttpResp* pRes, FILE* pOut, int64_t* pCopied, das_progress_f onProgress, + void* pUser +){ + DasErrCode nRet = DAS_OKAY; + int64_t nTotal = 0; + char buf[D2CHAR_CHUNK_SZ]; + int nRead = 0; + + while(true){ + nRead = _das_http_readChunk(pRes, buf, D2CHAR_CHUNK_SZ); + if(nRead == 0) break; + if(nRead < 0){ nRet = DASERR_HTTP; break; } + + errno = 0; + if(fwrite(buf, 1, nRead, pOut) != (size_t)nRead){ + daslog_error_v("Error writing message body, %s", strerror(errno)); + nRet = DASERR_IO; + break; + } + nTotal += nRead; + + /* The callback is the only way a caller can bail out of a long + * download, so a false return is honored even mid-body */ + if((onProgress != NULL) && !onProgress(nTotal, pUser)) break; + } + + _das_http_closeBody(pRes); + if(pCopied != NULL) *pCopied = nTotal; + return nRet; +} diff --git a/das3/http.h b/das3/http.h index 25f4d39..908a271 100644 --- a/das3/http.h +++ b/das3/http.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2017 Chris Piker +/* Copyright (C) 2017 - 2026 Chris Piker * * This file is part of das2C, the Core Das2 C Library. * @@ -19,6 +19,7 @@ #define _das_http_h_ #include +#include #include #include @@ -261,6 +262,47 @@ DAS_API DasAry* das_http_readUrl( int64_t nLimit, float rConSec ); +/** Progress callback for das_http_copyBody() + * + * @param nBytes The number of body bytes written so far + * @param pUser The opaque pointer handed to das_http_copyBody() + * @returns true to keep copying, false to stop. Stopping is the caller's + * decision, not a failure, see das_http_copyBody(). + */ +typedef bool (*das_progress_f)(int64_t nBytes, void* pUser); + +/** Copy the message body of an open HTTP response to a FILE + * + * The streaming counterpart to das_http_readUrl(). Call das_http_getBody() + * first, then hand the response here to drain the socket, or SSL connection, + * into pOut. Only one chunk is held in memory at a time so bodies of any + * size may be copied. + * + * The connection is closed before returning, whether or not the copy + * succeeded, so afterwards pRes->nSockFd is -1 and pRes->pSsl is NULL. + * The remaining response fields are untouched, release them with + * DasHttpResp_freeFields(). + * + * Errors are logged using the functions in log.h in the manner of + * das_http_readUrl(), das_error() is not called. + * + * @param pRes A response for which das_http_getBody() returned true + * @param pOut Destination for the body bytes. No flush is performed. + * @param pCopied If not NULL, receives the number of bytes written to pOut. + * Set even on failure or an early stop. + * @param onProgress If not NULL, called after each chunk is written. A false + * return stops the copy. + * @param pUser Passed through to onProgress untouched + * + * @returns DAS_OKAY if the whole body was copied or onProgress stopped the + * copy, DASERR_HTTP if reading from the server failed, DASERR_IO if + * writing to pOut failed. + */ +DAS_API DasErrCode das_http_copyBody( + DasHttpResp* pRes, FILE* pOut, int64_t* pCopied, das_progress_f onProgress, + void* pUser +); + /** @} */ #ifdef __cplusplus diff --git a/utilities/das_fetch.c b/utilities/das_fetch.c index bd2fae2..0069f31 100644 --- a/utilities/das_fetch.c +++ b/utilities/das_fetch.c @@ -1,6 +1,4 @@ /* Copyright (C) 2026 Chris Piker - * - * Author: Claude Fable 5.1 (Anthropic), from an idea by Chris Piker * * This file is part of das2C, the Core Das C Library. * @@ -99,7 +97,6 @@ #define URI_SZ 1024 #define URL_SZ 4096 #define ERR_SZ 1024 -#define CHUNK_SZ (64*1024) #define DEF_CACHE_AGE 3600 /* seconds a cached catalog node is trusted */ #define ROOT_ENV "DASFETCH_ROOT" @@ -2063,31 +2060,18 @@ static int streamUrl(DasCat* pCat, const char* sUrl, float rTimeout) das_stdio_binary(stdout); - DasIO* pIn = DasHttpResp_useSsl(&res) ? - new_DasIO_ssl(PROG, res.pSsl, "r") : new_DasIO_socket(PROG, res.nSockFd, "r"); - DasBuf* pBuf = new_DasBuf(CHUNK_SZ); - - int nRet = RET_OK; + /* The library closes the connection, success or not. Only the write + * failure is ours to classify, everything else is the server's fault + * as far as the exit code is concerned. */ int64_t nTotal = 0; - while(true){ - DasBuf_reinit(pBuf); - int nRead = DasIO_read(pIn, pBuf, CHUNK_SZ); - if(nRead < 0){ nRet = RET_SERVER; break; } - if(nRead == 0) break; - size_t uLen = 0; - const ubyte* pBytes = DasBuf_direct(pBuf, &uLen); - if(fwrite(pBytes, 1, uLen, stdout) != uLen){ - daslog_error_v("Write to standard output failed, %s", strerror(errno)); - nRet = RET_WRITE; - break; - } - nTotal += nRead; + int nRet = RET_OK; + switch(das_http_copyBody(&res, stdout, &nTotal, NULL, NULL)){ + case DAS_OKAY: nRet = RET_OK; break; + case DASERR_IO: nRet = RET_WRITE; break; + default: nRet = RET_SERVER; break; } fflush(stdout); - DasIO_close(pIn); - del_DasIO(pIn); - del_DasBuf(pBuf); DasHttpResp_freeFields(&res); daslog_info_v("%" PRId64 " bytes written", nTotal); diff --git a/utilities/validrange.h b/utilities/validrange.h index d407367..c4d1029 100644 --- a/utilities/validrange.h +++ b/utilities/validrange.h @@ -1,6 +1,4 @@ /* Copyright (C) 2026 Chris Piker - * - * Author: C. Piker, via Claude Fable 5.1 * * This file is part of das2C, the Core Das2 C Library. * From 364d8c7de905609a9d488ac7e738d69bcadc8042 Mon Sep 17 00:00:00 2001 From: Chris Piker Date: Sun, 25 Oct 2026 16:49:31 -0500 Subject: [PATCH 03/18] catalog: a DasCat session object, DasNode now opaque Lift the root policy, credentials and node memory that grew up inside das_fetch into das3/catalog.{c,h} so every client of libdas3 resolves catalog paths the same way. Catalog sub-trees may be mounted at points inside another catalog tree. * Doxygen category "catalog" * DasNode goes opaque * das3_node.c retired * Tests added Developed-with: Claude Fable 5.1 --- buildfiles/Darwin.mak | 2 + buildfiles/Linux.mak | 2 + buildfiles/Windows.mak | 2 + buildfiles/cmake/das2CTests.cmake | 1 + buildfiles/sources.mak | 7 +- das3/catalog.c | 452 ++++++++++++++++++++++++++++ das3/catalog.h | 318 ++++++++++++++++++++ das3/core.h | 2 +- das3/defs.h | 3 +- das3/dsdf.h | 4 - das3/http.c | 8 +- das3/http.h | 20 +- das3/json.h | 9 +- das3/node.c | 186 ++++++++---- das3/node.h | 183 +++++------- das3/node_priv.h | 233 +++++++++++++++ test/TestCat.c | 309 ++++++++++++++++++++ test/TestCatalog.c | 4 +- test/das_fetch/cat/geomag.json | 14 + test/das_fetch/cat/site.json | 14 + test/das_fetch/cat/uiowa.json | 24 ++ test/das_fetch_test.sh | 20 +- utilities/das3_node.c | 212 -------------- utilities/das_fetch.c | 471 +++++++++--------------------- 24 files changed, 1753 insertions(+), 747 deletions(-) create mode 100644 das3/catalog.c create mode 100644 das3/catalog.h create mode 100644 das3/node_priv.h create mode 100644 test/TestCat.c create mode 100644 test/das_fetch/cat/geomag.json create mode 100644 test/das_fetch/cat/site.json create mode 100644 test/das_fetch/cat/uiowa.json delete mode 100644 utilities/das3_node.c diff --git a/buildfiles/Darwin.mak b/buildfiles/Darwin.mak index 3f635bf..bb2787b 100644 --- a/buildfiles/Darwin.mak +++ b/buildfiles/Darwin.mak @@ -374,6 +374,8 @@ test_main: $(BD) $(BD)/$(TARG).a $(BUILD_TEST_PROGS) $(BULID_UTIL_PROGS) $(BD)/TestUri $(BD)/uri_tplt @echo "INFO: Running unit test for credentials manager, $(BD)/TestCredMngr..." @$(BD)/TestCredMngr $(BD) + @echo "INFO: Running unit test for catalog sessions over local nodes, $(BD)/TestCat..." + @$(BD)/TestCat @echo "INFO: Running unit test for exception names and send.h writers, $(BD)/TestOob..." @$(BD)/TestOob $(BD)/TestOob send 2 > $(BD)/oob_send.d2s && $(BD)/TestOob read $(BD)/oob_send.d2s diff --git a/buildfiles/Linux.mak b/buildfiles/Linux.mak index 8bf2dc3..951ec89 100644 --- a/buildfiles/Linux.mak +++ b/buildfiles/Linux.mak @@ -372,6 +372,8 @@ test_main: $(BD) $(BD)/$(TARG).a $(BUILD_TEST_PROGS) $(BUILD_UTIL_PROGS) $(BD)/TestUri $(BD)/uri_tplt @echo "INFO: Running unit test for credentials manager, $(BD)/TestCredMngr..." @$(BD)/TestCredMngr $(BD) + @echo "INFO: Running unit test for catalog sessions over local nodes, $(BD)/TestCat..." + @$(BD)/TestCat @echo "INFO: Running unit test for exception names and send.h writers, $(BD)/TestOob..." @$(BD)/TestOob $(BD)/TestOob send 2 > $(BD)/oob_send.d2s && $(BD)/TestOob read $(BD)/oob_send.d2s diff --git a/buildfiles/Windows.mak b/buildfiles/Windows.mak index 581bf41..ce6b65f 100644 --- a/buildfiles/Windows.mak +++ b/buildfiles/Windows.mak @@ -388,6 +388,8 @@ test_main: $(BD)\TestUri.exe $(BD)\uri_tplt @echo INFO: Running unit test for credentials manager, $(BD)\TestCredMngr... @$(BD)\TestCredMngr.exe $(BD) + @echo INFO: Running unit test for catalog sessions over local nodes, $(BD)\TestCat... + @$(BD)\TestCat.exe @echo INFO: Running unit test for exception names and send.h writers, $(BD)\TestOob... @$(BD)\TestOob.exe $(BD)\TestOob.exe send 2 > $(BD)\oob_send.d2s diff --git a/buildfiles/cmake/das2CTests.cmake b/buildfiles/cmake/das2CTests.cmake index 7d89442..969fc4e 100644 --- a/buildfiles/cmake/das2CTests.cmake +++ b/buildfiles/cmake/das2CTests.cmake @@ -64,6 +64,7 @@ _das_test(TestRaggedEncode main TestRaggedEncode _das_test(TestFilter main TestFilter) _das_test(TestUri main TestUri "${_bd}/uri_tplt") _das_test(TestCredMngr main TestCredMngr "${_bd}") +_das_test(TestCat main TestCat) _das_test(TestOob main TestOob) foreach(_v 2 3) if(_v EQUAL 2) diff --git a/buildfiles/sources.mak b/buildfiles/sources.mak index 7d5c6f6..ebc1525 100644 --- a/buildfiles/sources.mak +++ b/buildfiles/sources.mak @@ -15,6 +15,7 @@ LIB_SRCS = das3/das1.c das3/array.c das3/buffer.c das3/builder.c das3/cli.c \ das3/dataset_hdr2.c das3/dataset_hdr3.c das3/datum.c das3/descriptor.c \ das3/dft.c das3/dimension.c das3/dsdf.c das3/encoding.c das3/http.c \ das3/io.c das3/iterator.c das3/json.c das3/toml.c das3/log.c das3/node.c \ + das3/catalog.c \ das3/oob.c das3/operator.c das3/packet.c das3/plane.c das3/processor.c \ das3/property.c das3/send.c das3/stream.c das3/time.c das3/tt2000.c \ das3/units.c das3/utf8.c das3/util.c das3/value.c \ @@ -29,7 +30,7 @@ LIB_HDRS = das3/defs.h das3/time.h das3/das1.h das3/util.h das3/log.h \ das3/dimension.h das3/dataset.h das3/plane.h das3/packet.h das3/stream.h \ das3/processor.h das3/property.h das3/oob.h das3/io.h das3/iterator.h \ das3/builder.h das3/dsdf.h das3/credentials.h das3/http.h das3/dft.h \ - das3/json.h das3/toml.h das3/node.h das3/cli.h das3/send.h das3/uri.h \ + das3/json.h das3/toml.h das3/node.h das3/catalog.h das3/cli.h das3/send.h das3/uri.h \ das3/codec.h das3/codex.h das3/core.h das3/variable.h \ das3/form.h das3/form_rot.h das3/form_lin.h das3/form_pt.h \ das3/form_cplx.h das3/form_vec.h das3/form_pos.h das3/das2_compat.h \ @@ -50,7 +51,7 @@ UTIL_SRCS = utilities/das1_inctime.c utilities/das2_prtime.c \ utilities/das2_from_das1.c utilities/das2_from_tagged_das1.c \ utilities/das1_ascii.c utilities/das1_bin_avg.c utilities/das2_bin_ratesec.c \ utilities/das2_psd.c utilities/das2_hapi.c utilities/das2_histo.c \ - utilities/das2_cache_rdr.c utilities/das3_node.c utilities/das3_csv.c \ + utilities/das2_cache_rdr.c utilities/das3_csv.c \ utilities/das3_info.c utilities/das3_text.c utilities/das3_from_das2.c \ utilities/das_fetch.c @@ -60,7 +61,7 @@ CDF_UTIL_SRCS = utilities/das3_cdf.c utilities/das3_from_cdf.c \ utilities/das2_from_cdf.c TEST_SRCS = test/TestUnits.c test/TestArray.c test/TestDs.c test/TestBuilder.c \ - test/TestAuth.c test/TestCatalog.c test/TestTT2000.c test/ex_das_cli.c \ + test/TestAuth.c test/TestCatalog.c test/TestCat.c test/TestTT2000.c test/ex_das_cli.c \ test/ex_das_ephem.c test/TestCredMngr.c test/TestV3Read.c test/TestFeed.c \ test/TestProp.c \ test/TestIter.c test/TestUri.c test/TestFilter.c test/TestValue.c \ diff --git a/das3/catalog.c b/das3/catalog.c new file mode 100644 index 0000000..d2bd2e5 --- /dev/null +++ b/das3/catalog.c @@ -0,0 +1,452 @@ +/* Copyright (C) 2026 Chris Piker + * + * Author: C. Piker, via Claude Fable 5.1 + * + * This file is part of das2C, the Core Das2 C Library. + * + * Das2C is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License version 2.1 as published + * by the Free Software Foundation. + * + * Das2C is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for + * more details. + * + * You should have received a copy of the GNU Lesser General Public License + * version 2.1 along with das2C; if not, see . + */ + +/* A catalog session. Grew up as a pilot inside utilities/das_fetch.c and + * moved here once a second client wanted the same root policy. + * Policy misuse is reported with daslog_error and DASERR_CAT rather than + * das_error, because the usual source of a bad policy is a command line + * argument and a program should get to decide its own exit code for that. */ + +#define _POSIX_C_SOURCE 200112L + +#include +#include +#include + +#include "util.h" +#include "log.h" +#include "node_priv.h" +#include "catalog.h" + +struct das_cat_mount { + char* sUrl; + char* sUri; +}; + +struct das_cat { + DasCredMngr* pMgr; /* owned */ + char* sAgent; /* owned copy, or NULL for the library default */ + DasNodeCache* pCache; /* owned, NULL until DasCat_setCache() */ + + /* Root policy. Relative paths start at sPrefix in the federated + * catalog, unless a root is set, then at that node, or with mounts, at + * the first mounted node, or at the prefix if one was given. */ + char* sPrefix; /* never NULL */ + bool bPrefixGiven; + struct das_cat_mount root; /* the whole catalog; sUrl NULL if unset, + sUri NULL for a nameless root */ + struct das_cat_mount* aMounts; /* over the federated roots */ + size_t uMounts; + + DasNode* pRoot; /* memory root of the root and mount modes */ + DasNode** aRoots; /* every memory root made, freed with the session */ + size_t uRoots; + size_t uRootCap; + + char* sLastPath; /* the caller's path to the last resolve, or NULL */ + char* sLastUri; /* that path as the policy understood it */ +}; + +/* ************************************************************************* */ +/* Small helpers */ + +static bool _pfx(const char* s, const char* sPrefix) +{ + return (s != NULL) && (strncmp(s, sPrefix, strlen(sPrefix)) == 0); +} + +static void _setStr(char** ps, const char* s) +{ + free(*ps); + *ps = (s != NULL) ? das_strdup(s) : NULL; +} + +/* site:/x and test:/x are shorthand for the two well known children + * of the das2 root. new_RootNode() expands them too; doing it here as well + * keeps DasCat_lastUri() the full form. */ +static char* _expand(const char* sIn) +{ + if(_pfx(sIn, "site:")||_pfx(sIn, "test:")) + return das_string(D2URI_ROOT "%s", sIn); + return das_strdup(sIn); +} + +bool das_isUrlOrFile(const char* sPath) +{ + if(sPath == NULL) return false; + if(_pfx(sPath, "http://")||_pfx(sPath, "https://")||_pfx(sPath, "file://")) return true; + if((sPath[0] == '.')||(sPath[0] == '/')||(sPath[0] == '\\')) return true; + if(isalpha((unsigned char)sPath[0]) && (sPath[1] == ':') && ((sPath[2] == '\\')||(sPath[2] == '/'))) + return true; /* a Windows drive, catalog URIs never have their first colon at index 1 */ + size_t u = strlen(sPath); + return (u > 5) && (strcmp(sPath + u - 5, ".json") == 0); +} + +/* Take ownership of a memory root */ +static DasNode* _DasCat_keep(DasCat* pThis, DasNode* pRoot) +{ + if(pRoot == NULL) return NULL; + if(pThis->uRoots == pThis->uRootCap){ + size_t uCap = (pThis->uRootCap == 0) ? 8 : 2 * pThis->uRootCap; + DasNode** a = (DasNode**)realloc(pThis->aRoots, uCap * sizeof(DasNode*)); + if(a == NULL){ + das_error(DASERR_CAT, "Couldn't grow the catalog root list"); + return pRoot; /* leaks the root rather than losing the node */ + } + pThis->aRoots = a; + pThis->uRootCap = uCap; + } + pThis->aRoots[pThis->uRoots++] = pRoot; + return pRoot; +} + +/* ************************************************************************* */ +/* Construction */ + +DasCat* new_DasCat(DasCredMngr* pMgr, const char* sAgent) +{ + DasCat* pThis = (DasCat*)calloc(1, sizeof(DasCat)); + if(pThis == NULL){ + das_error(DASERR_CAT, "Couldn't allocate a catalog session"); + return NULL; + } + pThis->pMgr = pMgr; + if((sAgent != NULL) && (sAgent[0] != '\0')) pThis->sAgent = das_strdup(sAgent); + pThis->sPrefix = das_strdup(DASCAT_DEF_PREFIX); + return pThis; +} + +void del_DasCat(DasCat* pThis) +{ + if(pThis == NULL) return; + + /* Nodes borrow the credentials manager and the cache settings, so they go + * first. The policy root is in aRoots like every other root. */ + for(size_t u = 0; u < pThis->uRoots; ++u) del_RootNode(pThis->aRoots[u]); + free(pThis->aRoots); + if(pThis->pMgr != NULL) del_CredMngr(pThis->pMgr); + free(pThis->pCache); + + free(pThis->sAgent); + free(pThis->sPrefix); + free(pThis->root.sUrl); + free(pThis->root.sUri); + for(size_t u = 0; u < pThis->uMounts; ++u){ + free(pThis->aMounts[u].sUrl); + free(pThis->aMounts[u].sUri); + } + free(pThis->aMounts); + free(pThis->sLastPath); + free(pThis->sLastUri); + free(pThis); +} + +DasCredMngr* DasCat_credMngr(const DasCat* pThis){ return pThis->pMgr; } + +const char* DasCat_agent(const DasCat* pThis){ return pThis->sAgent; } + +void DasCat_setCache(DasCat* pThis, const char* sDir, int nMaxAgeSec) +{ + if(pThis->pCache == NULL) + pThis->pCache = (DasNodeCache*)calloc(1, sizeof(DasNodeCache)); + if(pThis->pCache == NULL){ + das_error(DASERR_CAT, "Couldn't allocate the node cache settings"); + return; + } + if((sDir == NULL)||(sDir[0] == '\0')){ + pThis->pCache->sDir[0] = '\0'; + return; + } + if(strlen(sDir) >= sizeof(pThis->pCache->sDir)){ + daslog_warn_v("Node cache directory name too long, caching is off: %s", sDir); + pThis->pCache->sDir[0] = '\0'; + return; + } + strcpy(pThis->pCache->sDir, sDir); + pThis->pCache->nMaxAgeSec = nMaxAgeSec; +} + +/* ************************************************************************* */ +/* Root policy */ + +/* Roots are built on first use, so the policy is frozen by then */ +static bool _DasCat_frozen(const DasCat* pThis) +{ + if(pThis->uRoots == 0) return false; + daslog_error("Catalog policy can't change after a path has been resolved"); + return true; +} + +DasErrCode DasCat_setPrefix(DasCat* pThis, const char* sUri) +{ + if(_DasCat_frozen(pThis)) return DASERR_CAT; + if((sUri == NULL)||(sUri[0] == '\0')){ + daslog_error("A path prefix can't be empty"); + return DASERR_CAT; + } + if(das_isUrlOrFile(sUri)){ + daslog_error_v("'%s' is a location, a path prefix is a catalog URI", sUri); + return DASERR_CAT; + } + free(pThis->sPrefix); + pThis->sPrefix = _expand(sUri); + pThis->bPrefixGiven = true; + return DAS_OKAY; +} + +const char* DasCat_prefix(const DasCat* pThis){ return pThis->sPrefix; } + +bool DasCat_hasRoot(const DasCat* pThis){ return pThis->root.sUrl != NULL; } + +/* The checks shared by roots and mounts */ +static bool _DasCat_locationOk(const char* sUrl, const char* sUri, bool bUriNeeded) +{ + if(!das_isUrlOrFile(sUrl)){ + daslog_error_v("A node is loaded from a URL or a file, '%s' is neither", sUrl ? sUrl : ""); + return false; + } + if((sUri == NULL)||(sUri[0] == '\0')){ + if(!bUriNeeded) return true; + daslog_error("A mount needs the catalog URI to put the node at"); + return false; + } + if(das_isUrlOrFile(sUri)){ + daslog_error_v("'%s' is a location, a catalog URI was expected", sUri); + return false; + } + return true; +} + +DasErrCode DasCat_setRoot(DasCat* pThis, const char* sUrl, const char* sUri) +{ + if(_DasCat_frozen(pThis)) return DASERR_CAT; + if(DasCat_hasRoot(pThis)){ + daslog_error("A catalog has one root, and one is already set"); + return DASERR_CAT; + } + if(pThis->uMounts > 0){ + daslog_error("A root replaces the federated catalog, which the mounts build on; use one or the other"); + return DASERR_CAT; + } + if(!_DasCat_locationOk(sUrl, sUri, false)) return DASERR_CAT; + _setStr(&pThis->root.sUrl, sUrl); + free(pThis->root.sUri); + pThis->root.sUri = ((sUri != NULL) && (sUri[0] != '\0')) ? _expand(sUri) : NULL; + return DAS_OKAY; +} + +const char* DasCat_rootUrl(const DasCat* pThis){ return pThis->root.sUrl; } + +const char* DasCat_rootUri(const DasCat* pThis){ return pThis->root.sUri; } + +DasErrCode DasCat_mount(DasCat* pThis, const char* sUrl, const char* sUri) +{ + if(_DasCat_frozen(pThis)) return DASERR_CAT; + if(DasCat_hasRoot(pThis)){ + daslog_error("A mount builds on the federated catalog, which a root replaces; use one or the other"); + return DASERR_CAT; + } + if(!_DasCat_locationOk(sUrl, sUri, true)) return DASERR_CAT; + + struct das_cat_mount* a = (struct das_cat_mount*)realloc( + pThis->aMounts, (pThis->uMounts + 1) * sizeof(struct das_cat_mount) + ); + if(a == NULL) return das_error(DASERR_CAT, "Couldn't grow the mount list"); + pThis->aMounts = a; + a[pThis->uMounts].sUrl = das_strdup(sUrl); + a[pThis->uMounts].sUri = _expand(sUri); + pThis->uMounts += 1; + return DAS_OKAY; +} + +size_t DasCat_mountCount(const DasCat* pThis){ return pThis->uMounts; } + +/* ************************************************************************* */ +/* Resolution */ + +/* Root node of the root and mount modes, loaded on first use. A named + * root is a mount root with one mount and no fall through. */ +static DasNode* _DasCat_root(DasCat* pThis) +{ + if(pThis->pRoot != NULL) return pThis->pRoot; + + if(pThis->root.sUrl != NULL){ + if(pThis->root.sUri == NULL){ + pThis->pRoot = _DasNode_newRootUrl( + pThis->root.sUrl, "", pThis->pMgr, pThis->sAgent, pThis->pCache + ); + } + else{ + const char* aUrl[1] = { pThis->root.sUrl }; + const char* aUri[1] = { pThis->root.sUri }; + pThis->pRoot = _DasNode_newMountRoot( + aUrl, aUri, 1, false, pThis->pMgr, pThis->sAgent, pThis->pCache + ); + } + } + else if(pThis->uMounts > 0){ + const char** aUrl = (const char**)calloc(pThis->uMounts, sizeof(char*)); + const char** aUri = (const char**)calloc(pThis->uMounts, sizeof(char*)); + if((aUrl == NULL)||(aUri == NULL)){ + das_error(DASERR_CAT, "Couldn't allocate the mount root"); + free(aUrl); free(aUri); + return NULL; + } + for(size_t u = 0; u < pThis->uMounts; ++u){ + aUrl[u] = pThis->aMounts[u].sUrl; + aUri[u] = pThis->aMounts[u].sUri; + } + pThis->pRoot = _DasNode_newMountRoot( + aUrl, aUri, pThis->uMounts, true, pThis->pMgr, pThis->sAgent, pThis->pCache + ); + free(aUrl); + free(aUri); + } + return _DasCat_keep(pThis, pThis->pRoot); +} + +DasNode* DasCat_resolve(DasCat* pThis, const char* sPath) +{ + bool bNoPath = (sPath == NULL)||(sPath[0] == '\0'); + _setStr(&pThis->sLastPath, bNoPath ? NULL : sPath); + free(pThis->sLastUri); + pThis->sLastUri = NULL; + + /* A node by location, named by itself */ + if(!bNoPath && das_isUrlOrFile(sPath)){ + pThis->sLastUri = das_strdup(sPath); + return _DasCat_keep(pThis, _DasNode_newRootUrl( + sPath, sPath, pThis->pMgr, pThis->sAgent, pThis->pCache + )); + } + + const char* sRootUri = pThis->root.sUri; + bool bMounted = (pThis->uMounts > 0); + if((sRootUri != NULL) || bMounted){ + DasNode* pRoot = _DasCat_root(pThis); + if(pRoot == NULL) return NULL; + const char* sTopUri = bMounted ? pThis->aMounts[0].sUri : sRootUri; + + /* No path: the named node itself. A full URI, or a relative path + * under an explicit prefix when the catalog is mounted over: the + * mount root decides whether a mount or the federated catalog owns + * it. Anything else is relative to the named node. */ + if(bNoPath){ + pThis->sLastUri = das_strdup(sTopUri); + return DasNode_subNode(pRoot, sTopUri, NULL, NULL); + } + if(bMounted && pThis->bPrefixGiven) + pThis->sLastUri = das_string("%s%s", pThis->sPrefix, sPath); + else + pThis->sLastUri = _expand(sPath); + if(_pfx(pThis->sLastUri, "tag:")||_pfx(pThis->sLastUri, sTopUri)) + return DasNode_subNode(pRoot, pThis->sLastUri, NULL, NULL); + + DasNode* pTop = DasNode_subNode(pRoot, sTopUri, NULL, NULL); + if(pTop == NULL){ + _setStr(&pThis->sLastUri, sTopUri); + return NULL; + } + const char* sSep = DasNode_isCatalog(pTop) ? DasNode_pathSep(pTop) : "/"; + free(pThis->sLastUri); + pThis->sLastUri = das_string("%s%s%s", sTopUri, sSep ? sSep : "/", sPath); + return DasNode_subNode(pTop, sPath, NULL, NULL); + } + + if(pThis->root.sUrl != NULL){ + DasNode* pRoot = _DasCat_root(pThis); + if(pRoot == NULL) return NULL; + if(bNoPath){ + pThis->sLastUri = das_strdup(pThis->root.sUrl); + return pRoot; + } + pThis->sLastUri = das_strdup(sPath); + return DasNode_subNode(pRoot, sPath, NULL, NULL); + } + + /* The federated catalog under the prefix */ + if(bNoPath){ + /* The prefix with its trailing separator removed names the container */ + pThis->sLastUri = das_strdup(pThis->sPrefix); + size_t u = strlen(pThis->sLastUri); + while((u > 0) && ((pThis->sLastUri[u-1] == '/')||(pThis->sLastUri[u-1] == ':'))) + pThis->sLastUri[--u] = '\0'; + } + else if(_pfx(sPath, "tag:")||_pfx(sPath, "site:")||_pfx(sPath, "test:")) + pThis->sLastUri = _expand(sPath); + else + pThis->sLastUri = das_string("%s%s", pThis->sPrefix, sPath); + + return _DasCat_keep(pThis, _DasNode_newRoot( + pThis->sLastUri, pThis->pMgr, pThis->sAgent, pThis->pCache + )); +} + +const char* DasCat_lastUri(const DasCat* pThis){ return pThis->sLastUri; } + +bool DasCat_covers(const DasCat* pThis, const char* sUri) +{ + /* Mounts fall through to the federated catalog and a nameless root has + * no URIs to be outside of, only a named root walls paths off */ + if(pThis->root.sUri == NULL) return true; + return _pfx(sUri, pThis->root.sUri); +} + +DasNode* DasCat_nearest(DasCat* pThis, const char** psRest) +{ + if(psRest != NULL) *psRest = NULL; + if((pThis->sLastPath == NULL) || das_isUrlOrFile(pThis->sLastPath)) return NULL; + + /* The resolves below overwrite the last path and URI, take them back + * so the caller's view of the failure survives a miss here */ + char* sOrigPath = pThis->sLastPath; + char* sOrigUri = pThis->sLastUri; + pThis->sLastPath = NULL; + pThis->sLastUri = NULL; + + char* sParent = das_strdup(sOrigPath); + DasNode* pNode = NULL; + size_t uCut = 0; + while(true){ + size_t u = strlen(sParent); + while((u > 0) && ((sParent[u-1] == '/')||(sParent[u-1] == ':'))) sParent[--u] = '\0'; + char* pLast = strrchr(sParent, '/'); + if((pLast == NULL)||(pLast == sParent)) break; + *pLast = '\0'; + uCut = (size_t)(pLast - sParent); /* sParent is a prefix copy of sOrigPath */ + DasNode* p = DasCat_resolve(pThis, sParent); + if((p != NULL) && DasNode_isCatalog(p)){ pNode = p; break; } + } + free(sParent); + free(pThis->sLastPath); + pThis->sLastPath = sOrigPath; + + if(pNode == NULL){ + free(pThis->sLastUri); + pThis->sLastUri = sOrigUri; + return NULL; + } + free(sOrigUri); + if(psRest != NULL){ + const char* s = sOrigPath + uCut; + while((*s == '/')||(*s == ':')) ++s; + *psRest = s; + } + return pNode; +} diff --git a/das3/catalog.h b/das3/catalog.h new file mode 100644 index 0000000..72033bd --- /dev/null +++ b/das3/catalog.h @@ -0,0 +1,318 @@ +/* Copyright (C) 2026 Chris Piker + * + * This file is part of das2C, the Core Das2 C Library. + * + * Das2C is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License version 2.1 as published + * by the Free Software Foundation. + * + * Das2C is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for + * more details. + * + * You should have received a copy of the GNU Lesser General Public License + * version 2.1 along with das2C; if not, see . + */ + +/** @file catalog.h A catalog session: roots, credentials and node memory + * in one object, with mountable sub-trees */ + +#ifndef _das_catalog_h_ +#define _das_catalog_h_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** @defgroup catalog Catalogs + * Finding data sources: the federated das catalog, catalog paths, and the + * objects that walk it. + * + * A das catalog is a tree of small JSON documents, mostly on web servers, + * leading from a well known root down to data source definitions. Each + * document is a node. Container nodes (Catalog, Collection, SourceSet) + * list their children by name along with the URLs each child can be loaded + * from. Leaf nodes (HttpStreamSrc, FileTimeAgg) describe how to get data. + * The tree is federated: the root at das2.org names a few top level trees + * and each site maintains its own branch. A path is resolved one network + * request per level, and only the levels on the way to the node asked for + * are ever loaded. + * + * Nodes are named by catalog path, which looks like a file path with a URI + * scheme in front: + * + * tag:das2.org,2012:site:/uiowa/juno/wav/survey + * + * In most programs the "tag:das2.org,2012:site:/" prefix is noise, so the + * library also accepts the relative form "uiowa/juno/wav/survey" and the + * shorthand "site:/uiowa/juno/wav/survey". A node can also be + * named by location, the URL or file it is stored at, which is how test + * catalogs and development servers are reached. + * + * To get a data source definition, open a catalog session, resolve a path + * and read the node: + * + * @code + DasCat* pCat = new_DasCat(NULL, "my_program/1.0"); + + DasNode* pNode = DasCat_resolve(pCat, "uiowa/juno/wav/survey"); + if((pNode == NULL) || !DasNode_isStreamSrc(pNode)) handle_error(); + + // The query interface of the source, laid out by the das catalog schema + const DasJdo* pProto = DasNode_getJdo(pNode, "protocol"); + ... + del_DasCat(pCat); // frees every node the session loaded + @endcode + * + * Two objects do the work. DasCat, in das3/catalog.h, is a session: it + * holds the root policy (where relative paths start, see DasCat_setRoot() + * and DasCat_mount()), the credentials and user agent every load uses, and + * the memory of every node it loads. DasNode, in das3/node.h, is one node: + * its type, path URI, source URL and parsed document, with DasNode_subNode() + * to step down to a child. Most programs only ever construct a DasCat. + * The node constructors in node.h are the lower level interface for + * programs that manage roots themselves; see "Working with nodes directly" + * below. + */ + +/** @addtogroup catalog + * @{ + */ + +/** The catalog URI prefix relative paths start from unless told otherwise */ +#define DASCAT_DEF_PREFIX "tag:das2.org,2012:site:/" + +/** A catalog session, opaque. + * + * A DasCat turns catalog paths into nodes. It remembers where relative + * paths start, which credentials and user agent to send, and every node it + * has loaded, so a program, a MATLAB extension or a Python module can say + * "resolve this path" and get a node back without managing roots itself. + * + * Paths given to DasCat_resolve() are catalog paths, not web paths, in any + * of these forms: + * + * - relative to the prefix: "uiowa/juno/wav/survey" + * - the shorthands "site:/uiowa/..." and "test:/..." + * - a full catalog URI: "tag:das2.org,2012:site:/uiowa/..." + * - a node by location, a URL or a file: "https://host/node.json", + * "./node.json" + * + * Where relative paths start is the root policy, and the catalog tree is + * navigated like a file system, so the policy borrows its words: + * + * - By default relative paths start at DASCAT_DEF_PREFIX in the + * federated catalog. DasCat_setPrefix() moves that. + * - DasCat_setRoot() makes one node, loaded by location, the whole + * catalog. Optionally the node is given its catalog URI, so paths + * under that URI resolve through it and DasNode_pathUri() reports + * full URIs. + * - DasCat_mount() puts a node, loaded by location, at a catalog URI + * on top of the federated catalog. Paths under the URI resolve + * through the mounted node, everything else through the compiled in + * roots as usual. Several nodes may be mounted. + * + * Policy is set before the first resolve. Nodes returned by + * DasCat_resolve() are owned by the catalog and live until del_DasCat(). + * + * @code + DasCat* pCat = new_DasCat(NULL, "my_program"); + DasNode* pNode = DasCat_resolve(pCat, "uiowa/juno/wav/survey"); + if(pNode == NULL){ + const char* sRest = NULL; + DasNode* pNear = DasCat_nearest(pCat, &sRest); + // pNear is the deepest catalog that did load, sRest what it lacks + } + del_DasCat(pCat); + @endcode + */ +typedef struct das_cat DasCat; + +/** Is this string a node location rather than a catalog path? + * + * True for http://, https:// and file:// URLs, for anything starting with + * '.', '/', '\' or a drive letter, and for anything ending in ".json". All else + * is taken to be a catalog path or URI. This is the rule DasCat_resolve() + * uses, published so command line parsers can agree with it. + * + * @memberof DasCat + */ +DAS_API bool das_isUrlOrFile(const char* sPath); + +/** Create a catalog session + * + * @param pMgr Credentials manager consulted when a node or data source asks + * for a login. May be NULL for public access only. Ownership + * passes to the catalog, del_DasCat() frees it. + * @param sAgent User agent string sent with every request, copied. NULL + * sends the library default. + * @returns a new catalog, free with del_DasCat() + * @memberof DasCat + */ +DAS_API DasCat* new_DasCat(DasCredMngr* pMgr, const char* sAgent); + +/** Free a catalog session, every node it loaded, and its credentials manager + * @memberof DasCat + */ +DAS_API void del_DasCat(DasCat* pThis); + +/** The credentials manager, borrowed, for data requests made by the caller + * @returns the manager given to new_DasCat(), or NULL + * @memberof DasCat + */ +DAS_API DasCredMngr* DasCat_credMngr(const DasCat* pThis); + +/** The user agent string, borrowed + * @returns the string given to new_DasCat(), or NULL + * @memberof DasCat + */ +DAS_API const char* DasCat_agent(const DasCat* pThis); + +/** Keep local copies of catalog nodes this session fetches over HTTP + * + * The per session form of DasNode_setCache(). Until called, nodes loaded + * through this catalog follow the process wide setting. Nodes read from + * local files are never cached. + * + * @param sDir Directory for the copies, created on first use. NULL or an + * empty string turns caching off for this session, whatever the + * process default says. + * @param nMaxAgeSec Copies older than this many seconds are fetched again. + * Zero or less means always fetch, but still save the result. + * @memberof DasCat + */ +DAS_API void DasCat_setCache(DasCat* pThis, const char* sDir, int nMaxAgeSec); + +/** Where relative paths start in the federated catalog + * + * The shorthands "site:" and "test:" are expanded. A URL or file is + * refused, those are roots, see DasCat_setRoot(). + * + * @param sUri A catalog URI prefix, typically ending in the separator the + * container uses, such as "tag:das2.org,2012:test:/" + * @returns DAS_OKAY, or DASERR_CAT with the reason logged + * @memberof DasCat + */ +DAS_API DasErrCode DasCat_setPrefix(DasCat* pThis, const char* sUri); + +/** The current prefix, DASCAT_DEF_PREFIX unless DasCat_setPrefix() was called + * @memberof DasCat + */ +DAS_API const char* DasCat_prefix(const DasCat* pThis); + +/** Make one node, loaded by location, the whole catalog + * + * Relative paths then start at that node and nothing outside it can be + * reached. Refused if a root is already set or anything is mounted. + * + * Without a URI the root has no catalog name: DasNode_pathUri() on nodes + * under it gives paths relative to the root, and a full URI given to + * DasCat_resolve() is just another relative path. With a URI the root is + * declared to be the node of that name, as a development server's copy of + * a catalog branch stands for the branch's federated name. Then full URIs + * under it resolve through it, nodes report full URIs, and URIs outside it + * fail, which DasCat_covers() can tell apart from an ordinary miss. + * + * @param sUrl A URL or file path that new_RootNode_url() accepts + * @param sUri The catalog URI the node has, shorthands accepted, or NULL + * for a nameless root + * @returns DAS_OKAY, or DASERR_CAT with the reason logged + * @memberof DasCat + */ +DAS_API DasErrCode DasCat_setRoot(DasCat* pThis, const char* sUrl, const char* sUri); + +/** The root location given to DasCat_setRoot(), or NULL if none is set + * @memberof DasCat + */ +DAS_API const char* DasCat_rootUrl(const DasCat* pThis); + +/** The root's catalog URI, or NULL if the root is nameless or not set + * @memberof DasCat + */ +DAS_API const char* DasCat_rootUri(const DasCat* pThis); + +/** Put a node, loaded by location, at a catalog URI over the federated catalog + * + * Paths under sUri load from sUrl and shadow whatever the federated + * catalog says for them, everything else resolves through the compiled in + * roots as usual. May be called more than once; when one mount point is + * under another the longer URI wins. Relative paths start at the first + * mount unless DasCat_setPrefix() was called, then they start at the + * prefix. Refused if a root is set. + * + * @param sUrl Where to load the node from, a URL or a file + * @param sUri The catalog URI to mount it at, shorthands accepted + * @returns DAS_OKAY, or DASERR_CAT with the reason logged + * @memberof DasCat + */ +DAS_API DasErrCode DasCat_mount(DasCat* pThis, const char* sUrl, const char* sUri); + +/** Number of nodes mounted with DasCat_mount() + * @memberof DasCat + */ +DAS_API size_t DasCat_mountCount(const DasCat* pThis); + +/** True if a root is set, so paths no longer reach the whole federated catalog + * @memberof DasCat + */ +DAS_API bool DasCat_hasRoot(const DasCat* pThis); + +/** Resolve a path to a node under the root policy + * + * Network access happens here, one request per catalog level not yet + * loaded. Nodes are kept, a second resolve of the same path costs nothing. + * + * @param sPath A catalog path in any of the forms listed for DasCat. NULL + * or empty means the root itself: the prefix container, the root + * node, or the first mounted node. + * @returns the node, owned by the catalog, or NULL if the path did not + * resolve. DasCat_lastUri() then gives the URI that was tried and + * DasCat_nearest() the deepest node that did load. + * @memberof DasCat + */ +DAS_API DasNode* DasCat_resolve(DasCat* pThis, const char* sPath); + +/** The full catalog URI of the last path given to DasCat_resolve(), as the + * policy understood it, resolved or not. For a path by location this is + * the location. + * @returns the URI, or NULL before the first resolve + * @memberof DasCat + */ +DAS_API const char* DasCat_lastUri(const DasCat* pThis); + +/** Can this URI be reached under the current policy? + * + * False only when a named root is set and sUri is not under its URI. A + * failed resolve of a URI this returns false for is a policy miss, not a + * catalog miss, and the message to the user should say so. + * @memberof DasCat + */ +DAS_API bool DasCat_covers(const DasCat* pThis, const char* sUri); + +/** After a failed resolve, the deepest catalog node on the path that did load + * + * Strips the last path component and resolves again, repeating until a + * container node loads or the path is exhausted. Costs network requests + * only for levels not already loaded. DasCat_lastUri() afterwards names + * the node returned, so a caller can say "no child X under URI Y" and list + * what Y does have with DasNode_subName(). + * + * @param psRest If not NULL, receives a pointer to the part of the failed + * path below the node returned, such as "survey/das2", valid until + * the next resolve. NULL when nothing is returned. + * @returns the node, owned by the catalog, or NULL if no part of the path + * resolves, the last resolve succeeded, or the path was a location. + * @memberof DasCat + */ +DAS_API DasNode* DasCat_nearest(DasCat* pThis, const char** psRest); + +/** @} */ + +#ifdef __cplusplus +} +#endif + +#endif /* _das_catalog_h_ */ diff --git a/das3/core.h b/das3/core.h index 737bb21..1c91aa6 100644 --- a/das3/core.h +++ b/das3/core.h @@ -232,7 +232,7 @@ das3/form_*.h Mathematical formalisms on composite types das3/utf8.h Counting characters instead of bytes das3/toml.h Reading TOML configuration files das3/dft.h Properly scaled power spectral densities -das3/node.h das catalog nodes, includes credentials.h and json.h +das3/catalog.h Finding data sources in the federated catalog system das3/http.h Reading streams from servers, credentials.h handles logins das3/builder.h Collect a whole das2 or das3 stream into DasDs objects das3/packet.h das2 packet descriptors, new code should use DasDs instead diff --git a/das3/defs.h b/das3/defs.h index da03cc3..b1db56c 100644 --- a/das3/defs.h +++ b/das3/defs.h @@ -269,7 +269,8 @@ typedef int DasErrCode; #define DASERR_URI 42 #define DASERR_FORM 43 #define DASERR_GEN 44 -#define DASERR_MAX 44 +#define DASERR_CAT 45 +#define DASERR_MAX 45 #ifdef __cplusplus } diff --git a/das3/dsdf.h b/das3/dsdf.h index de65510..586e170 100644 --- a/das3/dsdf.h +++ b/das3/dsdf.h @@ -27,9 +27,6 @@ extern "C" { #endif -/** @addtogroup catalog - * @{ - */ /** Parse a DSDF file into a descriptor object. @@ -101,7 +98,6 @@ DAS_API char* dsdf_valToNormParam( const char* sRawParam, char* sNormParam, size_t uNormLen ); -/** @} */ #ifdef __cplusplus } diff --git a/das3/http.c b/das3/http.c index b35e44c..bb80ab3 100644 --- a/das3/http.c +++ b/das3/http.c @@ -1328,8 +1328,12 @@ DasErrCode das_http_copyBody( nTotal += nRead; /* The callback is the only way a caller can bail out of a long - * download, so a false return is honored even mid-body */ - if((onProgress != NULL) && !onProgress(nTotal, pUser)) break; + * download, so its code is honored even mid-body and passed up + * unchanged for the caller to recognize */ + if(onProgress != NULL){ + nRet = onProgress(nTotal, pUser); + if(nRet != DAS_OKAY) break; + } } _das_http_closeBody(pRes); diff --git a/das3/http.h b/das3/http.h index 908a271..57bb251 100644 --- a/das3/http.h +++ b/das3/http.h @@ -263,13 +263,17 @@ DAS_API DasAry* das_http_readUrl( ); /** Progress callback for das_http_copyBody() + * + * Follows the StreamHandler convention: a non-zero return stops the copy + * and becomes the return value of das_http_copyBody(). Use a code of your + * own rather than DASERR_HTTP or DASERR_IO, otherwise a deliberate stop + * can't be told from a transport or write failure. * * @param nBytes The number of body bytes written so far * @param pUser The opaque pointer handed to das_http_copyBody() - * @returns true to keep copying, false to stop. Stopping is the caller's - * decision, not a failure, see das_http_copyBody(). + * @returns DAS_OKAY to keep copying, anything else to stop */ -typedef bool (*das_progress_f)(int64_t nBytes, void* pUser); +typedef DasErrCode (*das_progress_f)(int64_t nBytes, void* pUser); /** Copy the message body of an open HTTP response to a FILE * @@ -290,13 +294,13 @@ typedef bool (*das_progress_f)(int64_t nBytes, void* pUser); * @param pOut Destination for the body bytes. No flush is performed. * @param pCopied If not NULL, receives the number of bytes written to pOut. * Set even on failure or an early stop. - * @param onProgress If not NULL, called after each chunk is written. A false - * return stops the copy. + * @param onProgress If not NULL, called after each chunk is written. A + * non-zero return stops the copy. * @param pUser Passed through to onProgress untouched * - * @returns DAS_OKAY if the whole body was copied or onProgress stopped the - * copy, DASERR_HTTP if reading from the server failed, DASERR_IO if - * writing to pOut failed. + * @returns DAS_OKAY if the whole body was copied, DASERR_HTTP if reading + * from the server failed, DASERR_IO if writing to pOut failed, or + * whatever onProgress returned if it stopped the copy. */ DAS_API DasErrCode das_http_copyBody( DasHttpResp* pRes, FILE* pOut, int64_t* pCopied, das_progress_f onProgress, diff --git a/das3/json.h b/das3/json.h index dafad21..78caa80 100644 --- a/das3/json.h +++ b/das3/json.h @@ -66,7 +66,14 @@ extern "C" { #endif -/** @addtogroup catalog +/** @defgroup json JSON + * A parser for JSON text and accessors for the resulting document tree. + * + * Catalog nodes happen to be JSON documents, so DasNode_getJdo() returns + * these objects, but nothing here is specific to catalogs. + */ + +/** @addtogroup json * @{ */ diff --git a/das3/node.c b/das3/node.c index b8a3409..867e135 100644 --- a/das3/node.c +++ b/das3/node.c @@ -41,7 +41,7 @@ #include "json.h" /* Very nice lib from https://github.com/sheredom/json.h * customized for local use */ -#include "das3/node.h" +#include "node_priv.h" /* Time out values for connections */ @@ -93,7 +93,7 @@ static das_node_type_e das_node_type(const char* sType){ if(strcmp(sType, D2CV_TYPE_STREAM) == 0) return d2node_stream_src; if(strcmp(sType, D2CV_TYPE_TIMEAGG) == 0) return d2node_file_agg; if(strcmp(sType, D2CV_TYPE_SPASE) == 0) return d2node_spase_cat; - if(strcmp(sType, D2Cv_TYPE_SPDF_MASTER) == 0) return d2node_spdf_cat; + if(strcmp(sType, D2CV_TYPE_SPDF_MASTER) == 0) return d2node_spdf_cat; return d2node_inv; } @@ -149,7 +149,8 @@ const DasJdo* DasNode_getJdo(const DasNode* pThis, const char* sFragment) } -const DasJdo* DasNode_getJdoType( +/* Was public until 2026, nothing outside this file ever called it */ +static const DasJdo* _DasNode_getJdoType( const DasNode* pThis, enum das_json_type_e type, const char* sFragment ){ const DasJdo* pObj = DasNode_getJdo(pThis, sFragment); @@ -175,7 +176,7 @@ static const char* DasNode_rootStr(const DasNode* pThis, const char* sStr) /* Most root strings (title, description, version) are optional, so a miss is the caller's business, not an error. Callers that require a field check for NULL and complain themselves. */ - const DasJdo* pObj = DasNode_getJdoType(pThis, das_json_type_str, sStr); + const DasJdo* pObj = _DasNode_getJdoType(pThis, das_json_type_str, sStr); if(pObj == NULL) return NULL; return ((das_json_str*)pObj->value)->string; @@ -262,8 +263,8 @@ das_node_type_e DasNode_subType(const DasNode* pThis, size_t uIdx) /* Local file nodes. * * Catalog nodes are not only served over HTTP. Test fixtures, detached - * catalogs under development, and das2py's documented 'file://' URLs all - * want a node read straight from disk. Anything that is not an http(s) URL + * catalogs under development, and 'file://' URLs handed over by other + * tools all want a node read straight from disk. Anything that is not an http(s) URL * is treated as a path; the "file://" prefix is accepted and stripped. * Only the path forms "file:///abs/path" and "file://rel/path" are handled, * host names in file URLs are not. */ @@ -310,18 +311,19 @@ static DasAry* _DasNode_readFile(const char* sUrl) /* ************************************************************************* */ /* Node cache */ -static char g_sNodeCacheDir[256] = {'\0'}; -static int g_nNodeCacheAge = 0; +/* The process default. A DasCat carries its own settings and nodes it + * loads point there instead, see node_priv.h */ +static DasNodeCache g_defCache = { {'\0'}, 0 }; void DasNode_setCache(const char* sDir, int nMaxAgeSec) { if((sDir == NULL)||(sDir[0] == '\0')){ - g_sNodeCacheDir[0] = '\0'; + g_defCache.sDir[0] = '\0'; return; } - strncpy(g_sNodeCacheDir, sDir, sizeof(g_sNodeCacheDir) - 1); - g_sNodeCacheDir[sizeof(g_sNodeCacheDir) - 1] = '\0'; - g_nNodeCacheAge = nMaxAgeSec; + strncpy(g_defCache.sDir, sDir, sizeof(g_defCache.sDir) - 1); + g_defCache.sDir[sizeof(g_defCache.sDir) - 1] = '\0'; + g_defCache.nMaxAgeSec = nMaxAgeSec; } /* Cache file name for a URL: the URL minus its scheme with everything @@ -329,9 +331,11 @@ void DasNode_setCache(const char* sDir, int nMaxAgeSec) * that two URLs which sanitize alike still get separate files. Readable * names make an 'ls' of the cache directory useful when debugging a * catalog, a bare hash would not. */ -static bool _DasNode_cachePath(const char* sUrl, char* sPath, size_t uLen) -{ - if(g_sNodeCacheDir[0] == '\0') return false; +static bool _DasNode_cachePath( + const DasNodeCache* pCache, const char* sUrl, char* sPath, size_t uLen +){ + if(pCache == NULL) pCache = &g_defCache; + if(pCache->sDir[0] == '\0') return false; uint32_t uHash = 2166136261u; /* FNV-1a, no need for anything stronger */ for(const char* p = sUrl; *p != '\0'; ++p){ @@ -351,17 +355,18 @@ static bool _DasNode_cachePath(const char* sUrl, char* sPath, size_t uLen) } sName[u] = '\0'; - int n = snprintf(sPath, uLen, "%s%s%s_%08" PRIx32 ".json", g_sNodeCacheDir, DAS_DSEPS, sName, uHash); + int n = snprintf(sPath, uLen, "%s%s%s_%08" PRIx32 ".json", pCache->sDir, DAS_DSEPS, sName, uHash); return (n > 0) && ((size_t)n < uLen); } /* A fresh enough cached copy, or NULL */ -static DasAry* _DasNode_readCache(const char* sPath) +static DasAry* _DasNode_readCache(const DasNodeCache* pCache, const char* sPath) { - if(g_nNodeCacheAge <= 0) return NULL; + if(pCache == NULL) pCache = &g_defCache; + if(pCache->nMaxAgeSec <= 0) return NULL; struct stat st; if(stat(sPath, &st) != 0) return NULL; - if((time(NULL) - st.st_mtime) > g_nNodeCacheAge) return NULL; + if((time(NULL) - st.st_mtime) > pCache->nMaxAgeSec) return NULL; return _DasNode_readFile(sPath); } @@ -507,7 +512,7 @@ static DasNode* _DasNode_fromJson( */ static DasNode* _DasNode_mkNode( const char* sUrl, const char* sPathUri, DasCredMngr* pMgr, - const char* sAgent, float rConSec + const char* sAgent, float rConSec, const DasNodeCache* pCache ){ /* Save the final URL, may be needed for error messages, may not match * sUrl due to re-directs */ @@ -520,8 +525,8 @@ static DasNode* _DasNode_mkNode( pBytesAry = _DasNode_readFile(sUrl); strncpy(sFinalUrl, sUrl, sizeof(sFinalUrl) - 1); } - else if(_DasNode_cachePath(sUrl, sCache, sizeof(sCache)) && - ((pBytesAry = _DasNode_readCache(sCache)) != NULL)){ + else if(_DasNode_cachePath(pCache, sUrl, sCache, sizeof(sCache)) && + ((pBytesAry = _DasNode_readCache(pCache, sCache)) != NULL)){ daslog_debug_v("Node %s read from cache", sUrl); strncpy(sFinalUrl, sUrl, sizeof(sFinalUrl) - 1); bFromCache = true; @@ -547,7 +552,7 @@ static DasNode* _DasNode_mkNode( size_t uBytes; const char* pBytes = (const char*) (DasAry_getBytesIn(pBytesAry, DIM0, &uBytes)); DasNode* pNode = _DasNode_fromJson(pBytes, uBytes, sUrl, sFinalUrl, sPathUri); - if(pNode != NULL){ pNode->pMgr = pMgr; pNode->sAgent = sAgent; } + if(pNode != NULL){ pNode->pMgr = pMgr; pNode->sAgent = sAgent; pNode->pCache = pCache; } /* Only documents that parse are worth keeping, a server error page is not */ if((pNode != NULL) && (sCache[0] != '\0') && !bFromCache) @@ -621,12 +626,12 @@ static bool _checkSubMatch(DasNode* pBase, const char* sSubPathWithSep) const char* sSep = "/"; - const DasJdo* pObj = DasNode_getJdoType(pBase, das_json_type_str, D2FRAG_PATH_SEP); + const DasJdo* pObj = _DasNode_getJdoType(pBase, das_json_type_str, D2FRAG_PATH_SEP); if(pObj != NULL){ sSep = ((das_json_str*)pObj->value)->string; } else{ - pObj = DasNode_getJdoType(pBase, das_json_type_null, D2FRAG_PATH_SEP); + pObj = _DasNode_getJdoType(pBase, das_json_type_null, D2FRAG_PATH_SEP); if(pObj != NULL) sSep = ""; } @@ -749,7 +754,7 @@ static DasNode* _DasNode_loadSubNode_dasCat( * so that we know if we can try again with a slow server, of if * we just received bad data and shouldn't waste time reading it * again */ - pNode = _DasNode_mkNode(sUrl, sSubUri, pMgr, sAgent, rConSec); + pNode = _DasNode_mkNode(sUrl, sSubUri, pMgr, sAgent, rConSec, pThis->base.pCache); if(pNode == NULL) continue; /* If after removing the child portion from the relative path there @@ -949,15 +954,22 @@ void del_RootNode(DasNode* pThis) /* ************************************************************************* */ /* Root Node Construction */ -DasNode* new_RootNode_url( - const char* sUrl, const char* sPathUri, DasCredMngr* pMgr, const char* sAgent +DasNode* _DasNode_newRootUrl( + const char* sUrl, const char* sPathUri, DasCredMngr* pMgr, const char* sAgent, + const DasNodeCache* pCache ){ /* Since they want a specific URL, go for the max timeout */ - DasNode* pNode = _DasNode_mkNode(sUrl, sPathUri, pMgr, sAgent, DASHTTP_TO_MAX); + DasNode* pNode = _DasNode_mkNode(sUrl, sPathUri, pMgr, sAgent, DASHTTP_TO_MAX, pCache); if(pNode) pNode->bIsRoot = true; return pNode; } +DasNode* new_RootNode_url( + const char* sUrl, const char* sPathUri, DasCredMngr* pMgr, const char* sAgent +){ + return _DasNode_newRootUrl(sUrl, sPathUri, pMgr, sAgent, NULL); +} + /* Append a JSON string literal, escaping the two characters that matter */ static size_t _jsonStr(char* sOut, size_t uOut, const char* sIn) { @@ -972,67 +984,111 @@ static size_t _jsonStr(char* sOut, size_t uOut, const char* sIn) return u; } -DasNode* new_RootNode_seed( - const char* sUri, const char* sUrl, bool bFallThrough, DasCredMngr* pMgr, - const char* sAgent +/* read comments in node_priv.h header on this function first before edititing it */ +DasNode* _DasNode_newMountRoot( + const char** aUrl, const char** aUri, size_t uMounts, + bool bFallThrough, DasCredMngr* pMgr, const char* sAgent, + const DasNodeCache* pCache ){ - if((sUri == NULL)||(sUri[0] == '\0')||(sUrl == NULL)||(sUrl[0] == '\0')){ - daslog_error("A catalog seed needs both a URI and a URL"); + if(uMounts == 0){ + daslog_error("A mount root needs at least one mount"); return NULL; } + size_t uTotal = 0; + for(size_t i = 0; i < uMounts; ++i){ + if((aUri[i] == NULL)||(aUri[i][0] == '\0')||(aUrl[i] == NULL)||(aUrl[i][0] == '\0')){ + daslog_error("A mount needs both a URL to load and a URI to sit at"); + return NULL; + } + uTotal += strlen(aUri[i]) + strlen(aUrl[i]); + } - /* The seed root is an ordinary Catalog node that only exists in memory. + /* The mount root is an ordinary Catalog node that only exists in memory. * Its separator is null, like the federated root's, so child names are - * whole URI prefixes. The seed comes first, then, if asked for, a child + * whole URI prefixes. The mounts come first, then, if asked for, a child * with the empty name whose URLs are the federated roots. The empty name - * is a prefix of every path, so anything the seed does not claim falls - * through to the normal catalog, while everything under the seed URI is - * shadowed. Without that child the seed is all there is. */ - char sJson[4096]; + * is a prefix of every path, so anything no mount claims falls through + * to the normal catalog, while everything under a mount URI is shadowed. + * Without that child the mounts are all there is. + * + * Sized from the inputs since a mount list has no natural bound. The + * escaping below at most doubles a string, hence the factor. */ + size_t uJson = 256 + 2 * uTotal + 64 * uMounts; + for(int i = 0; i < g_nDasDefRoots; ++i) uJson += 2 * strlen(g_lDasDefRoots[i]) + 4; + char* sJson = (char*)calloc(uJson, 1); + if(sJson == NULL){ + das_error(DASERR_NODE, "Couldn't allocate mount root"); + return NULL; + } size_t u = 0; - u += snprintf(sJson + u, sizeof(sJson) - u, - "{\"type\":\"Catalog\",\"version\":\"0.5\",\"name\":\"seed\"," - "\"title\":\"Seeded catalog root\",\"separator\":null,\"catalog\":{" + u += snprintf(sJson + u, uJson - u, + "{\"type\":\"Catalog\",\"version\":\"0.5\",\"name\":\"mounts\"," + "\"title\":\"Mount root\",\"separator\":null,\"catalog\":{" ); - u += _jsonStr(sJson + u, sizeof(sJson) - u, sUri); - u += snprintf(sJson + u, sizeof(sJson) - u, ":{\"type\":\"Catalog\",\"urls\":["); - u += _jsonStr(sJson + u, sizeof(sJson) - u, sUrl); - u += snprintf(sJson + u, sizeof(sJson) - u, "]}"); + + /* Longest URI first, see node_priv.h. A selection sort over an index + * array; mount lists are a handful of entries */ + size_t aOrder[64]; + if(uMounts > sizeof(aOrder)/sizeof(aOrder[0])){ + daslog_error("Too many mounts"); + free(sJson); + return NULL; + } + for(size_t i = 0; i < uMounts; ++i) aOrder[i] = i; + for(size_t i = 0; i + 1 < uMounts; ++i){ + size_t iMax = i; + for(size_t j = i + 1; j < uMounts; ++j) + if(strlen(aUri[aOrder[j]]) > strlen(aUri[aOrder[iMax]])) iMax = j; + size_t t = aOrder[i]; aOrder[i] = aOrder[iMax]; aOrder[iMax] = t; + } + + for(size_t i = 0; i < uMounts; ++i){ + size_t k = aOrder[i]; + if(i > 0) u += snprintf(sJson + u, uJson - u, ","); + u += _jsonStr(sJson + u, uJson - u, aUri[k]); + u += snprintf(sJson + u, uJson - u, ":{\"type\":\"Catalog\",\"urls\":["); + u += _jsonStr(sJson + u, uJson - u, aUrl[k]); + u += snprintf(sJson + u, uJson - u, "]}"); + } if(bFallThrough){ - u += snprintf(sJson + u, sizeof(sJson) - u, ",\"\":{\"type\":\"Catalog\",\"urls\":["); + u += snprintf(sJson + u, uJson - u, ",\"\":{\"type\":\"Catalog\",\"urls\":["); for(int i = 0; i < g_nDasDefRoots; ++i){ - if(i > 0) u += snprintf(sJson + u, sizeof(sJson) - u, ","); - u += _jsonStr(sJson + u, sizeof(sJson) - u, g_lDasDefRoots[i]); + if(i > 0) u += snprintf(sJson + u, uJson - u, ","); + u += _jsonStr(sJson + u, uJson - u, g_lDasDefRoots[i]); } - u += snprintf(sJson + u, sizeof(sJson) - u, "]}"); + u += snprintf(sJson + u, uJson - u, "]}"); } - u += snprintf(sJson + u, sizeof(sJson) - u, "}}"); - if(u >= sizeof(sJson) - 1){ - daslog_error("Catalog seed URI or URL too long"); + u += snprintf(sJson + u, uJson - u, "}}"); + if(u >= uJson - 1){ + daslog_error("Mount URI or URL too long"); + free(sJson); return NULL; } /* Nothing is fetched here, the first subNode call does that and takes - the manager and agent from the root */ - DasNode* pRoot = _DasNode_fromJson(sJson, u, "memory:seed", "memory:seed", ""); + the manager, agent and cache from the root */ + DasNode* pRoot = _DasNode_fromJson(sJson, u, "memory:mounts", "memory:mounts", ""); + free(sJson); if(pRoot != NULL){ pRoot->bIsRoot = true; pRoot->pMgr = pMgr; pRoot->sAgent = sAgent; + pRoot->pCache = pCache; } return pRoot; } -DasNode* new_RootNode( - const char* sPathUri, DasCredMngr* pMgr, const char* sAgent +DasNode* _DasNode_newRoot( + const char* sPathUri, DasCredMngr* pMgr, const char* sAgent, + const DasNodeCache* pCache ){ /* Find the node using the path, start at the root */ DasNode* pTop = NULL; DasNode* pNode = NULL; - /* das2py accepts "site:/..." and "test:/..." as shorthand for the two - well known children of the das2 root. Do the same here so every - client gets it, and so the two libraries agree on what a path means. */ + /* "site:/..." and "test:/..." are shorthand for the two well known + children of the das2 root, see D2URI_SITES. Expanding them here, in + the library, means every client gets the same spelling. */ char sFullUri[600] = {'\0'}; if((sPathUri != NULL) && ( (strncmp(sPathUri, "site:", 5) == 0) || (strncmp(sPathUri, "test:", 5) == 0) @@ -1051,7 +1107,7 @@ DasNode* new_RootNode( while((rConSec <= DASHTTP_TO_MAX )&&(pNode == NULL)){ for(int i = 0; i < g_nDasDefRoots; ++i){ /* Credentials should never be needed for top level nodes */ - pTop = _DasNode_mkNode(g_lDasDefRoots[i], NULL, NULL, sAgent, rConSec); + pTop = _DasNode_mkNode(g_lDasDefRoots[i], NULL, NULL, sAgent, rConSec, pCache); if(pTop == NULL) continue; if(pTop->nType != d2node_catalog) continue; @@ -1080,8 +1136,14 @@ DasNode* new_RootNode( pNode->pMgr = pMgr; /* pTop was fetched without them, the kept node hands them to whatever it loads later */ pNode->sAgent = sAgent; + pNode->pCache = pCache; del_NonRootNode(pTop); return pNode; } +DasNode* new_RootNode(const char* sPathUri, DasCredMngr* pMgr, const char* sAgent) +{ + return _DasNode_newRoot(sPathUri, pMgr, sAgent, NULL); +} + diff --git a/das3/node.h b/das3/node.h index 1afb21d..74e300d 100644 --- a/das3/node.h +++ b/das3/node.h @@ -28,27 +28,32 @@ extern "C" { -/** @defgroup catalog Catalogs - * Provides utilities for reading remote data source catalogs and caching their - * contents in RAM. - * - * There are two basic methods of catalog object acquisition, finding a node by - * it's path from the global root of the federated das2 catalog system: +/** @addtogroup catalog + * + * @par Working with nodes directly + * + * The node constructors in this header are what DasCat is built on. Use + * them when a program manages its own roots, for instance to hold two + * unrelated catalogs at once, or to load a single isolated node from a URL + * and nothing else. + * + * There are two basic methods of node acquisition, finding a node by + * its path from the global root of the federated das2 catalog system: * (ex: tag:das2.org,2012:site:/swri/mars_express/aspera3/sciels) - * or by an loading an explicit URL + * or by loading an explicit URL * (ex: http://das2.org/catalog/das/site/uiowa/cassini/rpws/survey.json). * * When describing the das2 catalog C-API there are two meanings for the word * Root. * - * 1) Memory Roots - A node can be the in-memory root of a linked list of nodes. + * 1) Memory Roots - A node can be the in-memory root of a tree of nodes. * * 2) Global Root - Root can also refer to the global das2 top-level un-named - * node which resides at http://das2.org/catalog/index.json and it's mirrors. + * node which resides at http://das2.org/catalog/index.json and its mirrors. * * Any node loaded using new_RootNode() or new_RootNode_url() is an *in memory* * root node. It might not be the root of a federated catalog system, but it - * is still a top level memory item that manages a memory cache for it's + * is still a top level memory item that manages a memory cache for its * subnodes. If new_RootNode() happens to be called with a URI of 'NULL' then * the in-memory root corresponds to the federated catalog root. The reason for * splitting the concept of the in-memory root from the global catalog root @@ -69,29 +74,29 @@ extern "C" { // were traversed to get to the requested item are deleted from memory. DasNode* pRoot = new_RootNode("tag:das2.org,2012:site:/uiowa", NULL, NULL); - if(!DasNode_isCatalog(pNode)) handle_error(); + if(!DasNode_isCatalog(pRoot)) handle_error(); // Use the site catalog to get the Cassini Saturn SLS2 data source description - const char* sRelPath = "uiowa/cassini/ephemeris/saturn_sls2"; + const char* sRelPath = "cassini/ephemeris/saturn_sls2"; DasNode* pNode = DasNode_subNode(pRoot, sRelPath, NULL, NULL); - if(!(DasNode_isStreamSource(pNode))) handle_error(); + if(!DasNode_isStreamSrc(pNode)) handle_error(); - // Get the HTTP get Query interface definition + // Get the HTTP GET protocol definition - const DasJdo* pDef = DasNode_getJdo(pNode, "SOURCE/QUERY_PARAMS"); + const DasJdo* pDef = DasNode_getJdo(pNode, "protocol"); if(pDef == NULL) handle_error(); // Freeing the root node will delete all lower nodes - del_DasNode(pRoot); + del_RootNode(pRoot); @endcode * - * You now have a JSON object providing the query definition see - * http://das2.org/datasource.html for more information on the content of das2 - * data source definitions. + * You now have a JSON object describing how the source is queried. The + * layout of HttpStreamSrc nodes is given by the das catalog schema, see + * DasNode_version() for the two layouts in the wild. * * For isolated sites, or for testing purposes catalog nodes can be loaded * directly. The following example acquires a local test root and then a @@ -110,12 +115,13 @@ extern "C" { CredMngr_addCred(pMngr, &cred); const char* sTestUrl = "http://my.domain.com/das2/regtest.json"; - DasNode* pRoot = new_RootNode_url(sTestUrl, pMngr, NULL); - if(!DasNode_isCatalog(pNode)) handle_error(); + DasNode* pRoot = new_RootNode_url(sTestUrl, "regtest", pMngr, NULL); + if(!DasNode_isCatalog(pRoot)) handle_error(); - // At this point sub-items can be loaded from the local root as above. + // At this point sub-items can be loaded from the local root as above. The + // credentials manager given to the root is inherited by the lookup. - DasNode* pNode = DasNode_subNode("MarsExpress/aspera/els", pMngr, NULL); + DasNode* pNode = DasNode_subNode(pRoot, "MarsExpress/aspera/els", NULL, NULL); @endcode * @@ -146,7 +152,7 @@ extern "C" { #define D2CV_TYPE_STREAM "HttpStreamSrc" #define D2CV_TYPE_TIMEAGG "FileTimeAgg" #define D2CV_TYPE_SPASE "SpaseRecord" -#define D2Cv_TYPE_SPDF_MASTER "SpdfMasterCat" +#define D2CV_TYPE_SPDF_MASTER "SpdfMasterCat" /** Catalog node type */ typedef enum das_node_type_enum { @@ -159,23 +165,20 @@ typedef enum das_node_type_enum { * @{ */ -/** Base type for das2 catalog nodes. */ -typedef struct das_node { - das_node_type_e nType; /* Holds the node type */ - char sURL[512]; /* Holds source URL for this node */ - char sPath[512]; /* Holds the Path URI for this node */ - bool bIsRoot; /* True if a local root node (i.e. is a memory manager) */ - void* pDom; /* A pointer to the document tokens. */ - DasCredMngr* pMgr; /* Credentials used to load this node, inherited by - children. Borrowed, the caller keeps it alive. */ - const char* sAgent; /* User agent used to load this node, inherited by - children. Borrowed, usually a string literal. */ -} DasNode; +/** A das catalog node, opaque. + * + * Nodes are made by the new_RootNode*() constructors and DasNode_subNode(), + * read through the accessors below, and freed by del_RootNode() on the + * root that loaded them. There is no public layout: the document, the + * source URL and the path URI are reached with DasNode_getJdo(), + * DasNode_srcUrl() and DasNode_pathUri(). + */ +typedef struct das_node DasNode; /** @} */ /** Return the compiled in catalog root URLs - * @param pLen - A pointer to a size_t to received the number of builtin roots + * @param pLen - A pointer to a size_t to receive the number of builtin roots * @return A pointer to an array of null terminated strings. */ DAS_API const char** das_root_urls(size_t* pLen); @@ -194,25 +197,31 @@ DAS_API const char** das_root_urls(size_t* pLen); DasNode* pRoot = NULL; // Get the das2 site root catalog node - pRoot = new_RootNode(D2URI_DAS_SITE_ROOT, NULL, NULL, NULL); + pRoot = new_RootNode(D2URI_SITES, NULL, NULL); // Get the U. Iowa site catalog top node - pRoot = new_RootNode(D2URI_DAS_SITE_ROOT "/uiowa", NULL, NULL); + pRoot = new_RootNode(D2URI_SITE_UIOWA, NULL, NULL); @endcode * * @param sPathUri Retrieve a catalog node given a global path URI. This a * location from the top of the federated catalog system, not - * a file system or web path. - * - * @param sAgent The user agent string you wish to send to the server. If - * NULL then the string "libdas/3" is sent. + * a file system or web path. NULL loads the federated root + * itself. The shorthands "site:/..." and "test:/..." + * are accepted for the two well known children of the das2 + * root. * * @param pMgr A credentials manager object to consult if a password is * requested. May be set to NULL to indicate that only public * items may be requested. Usually this is not needed as most * catalog entries are public though the data sources they - * describe may not be. + * describe may not be. The returned node remembers it and + * hands it to anything loaded through DasNode_subNode(). + * + * @param sAgent The user agent string you wish to send to the server. If + * NULL then the string "libdas/3" is sent. Borrowed, not + * copied, so a string literal or a buffer that outlives the + * node. * * @return A new DasNode object allocated on the heap, or NULL if object * resolution failed. Calling del_RootNode() for the returned @@ -239,79 +248,50 @@ DAS_API DasNode* new_RootNode( DasNode* pRoot = NULL; // Get a standalone local Voyager catalog root - pRoot = new_RootNode(NULL, "http://my.site.gov/test/local_vgr.json", NULL, NULL); + pRoot = new_RootNode_url("http://my.site.gov/test/local_vgr.json", "vgr", NULL, NULL); // Get a standalone data source description for Voyager PLS data - pRoot = new_RootNode(NULL, "http://my.site.gov/test/local_vgr/pls.json", NULL, NULL); + pRoot = new_RootNode_url("http://my.site.gov/test/local_vgr/pls.json", "vgr/pls", NULL, NULL); + + // Nodes need not come from a server, a path or file:// URL reads a file + pRoot = new_RootNode_url("./local_vgr.json", "vgr", NULL, NULL); @endcode * - * @param sUrl Retrieve a catalog node from an explicit Url. No searches - * will be preformed. + * @param sUrl Retrieve a catalog node from an explicit URL. No searches + * will be performed. Anything that is not an http(s) URL is + * read as a local file, with or without a "file://" prefix. * * @param sPathUri Since the node was not loaded by following the federated - * catalog system to an end point, you will have to tell the node - * it's name. Any name will worked and it's saved internally. - * - * @param sAgent The user agent string you wish to send to the server. If - * NULL then the string "libdas/3" is sent. + * catalog system to an end point, you have to tell the node + * its name. Any name works, it is saved and reported by + * DasNode_pathUri(), and children get it as a prefix. May + * be the empty string for a root that has no name. * * @param pMgr A credentials manager object to consult if a password is * requested. May be set to NULL to indicate that only public - * items may be requested. Usually this is not needed as most - * catalog entries are public though the data sources they - * describe may not be. + * items may be requested. Inherited by DasNode_subNode(). + * + * @param sAgent The user agent string you wish to send to the server. If + * NULL then the string "libdas/3" is sent. Borrowed, not + * copied. * * @return A new DasNode object allocated on the heap, or NULL if object - * resolution failed. Use DasNode_type() to determine the which - * type of Catalog item was acquired. + * resolution failed. Use DasNode_type() to determine which + * type of catalog item was acquired. * * @memberof DasNode */ DAS_API DasNode* new_RootNode_url(const char* sUrl, const char* sPathUri, DasCredMngr* pMgr, const char* sAgent); -/** Make a catalog root that names one node by URI and loads it from a URL. - * - * A seed states, out of band, that the node at sUrl is the one named sUri, - * for example that a development server's copy of a catalog branch stands - * for the branch's federated name, or that a DOI is the URI of a catalog - * kept outside the das2.org tag scheme. Paths under the seed URI load - * from the seeded URL and shadow whatever the federated catalog says for - * them. What happens to paths outside the seed URI is up to bFallThrough: - * true patches the federated catalog, so those paths resolve through the - * compiled in roots as usual; false makes the seed the only root, so they - * fail to resolve. - * - * Nothing is fetched here. The returned root is an in memory Catalog node, - * use DasNode_subNode() on it with sUri or any path to load nodes, and - * del_RootNode() to free everything loaded through it. Nodes loaded this - * way report sUri based path URIs from DasNode_pathUri(). - * - * @param sUri The catalog URI the seeded node has, such as - * "tag:das2.org,2012:site:/uiowa/tracers/tracers-1" - * @param sUrl Where to load that node from, any URL or file path that - * new_RootNode_url() accepts - * @param bFallThrough If true paths outside sUri resolve through the - * federated roots, if false the seed is the whole catalog - * @param pMgr Credentials manager for nodes loaded through this root, may - * be NULL. DasNode_subNode() inherits it unless given another. - * @param sAgent User agent string for the same, may be NULL - * @return A new root node, or NULL if the arguments are unusable - * @memberof DasNode - */ -DAS_API DasNode* new_RootNode_seed( - const char* sUri, const char* sUrl, bool bFallThrough, DasCredMngr* pMgr, - const char* sAgent -); - /** Get a das2 catalog node contained item. * * This function will search down the catalog hierarchy and find the requested * node, downloading any intermediate nodes as needed. If the descendant node * has already been downloaded no new network activity is generated. * - * @param pThis A pointer to a node of type D2C_CATALOG + * @param pThis A container node, one for which DasNode_isCatalog() is true * * @param sRelPath the ID of the descendant node. This id is the portion * that would be appended to this node's PathUri to make the @@ -505,20 +485,7 @@ DAS_API bool DasNode_isJson(const DasNode* pThis); DAS_API const DasJdo* DasNode_getJdo(const DasNode* pThis, const char* sFragment); -/** Get a JSON document object of a particular type at a fragment location - * @param pThis The catalog node - * @param type The type of object expected - * @param sFragment The fragment path, for example "CONTACTS/TECH/0/EMAIL" - * @return The json object at that location, or NULL if the node does not - * contain the given element, or if the given element is of the - * wrong type - * @memberof DasNode - */ -DAS_API const DasJdo* DasNode_getJdoType( - const DasNode* pThis, enum das_json_type_e, const char* sFragment -); - -/** Delete a root node freeing it's memory. +/** Delete a root node freeing its memory. * All sub-nodes will be deleted as well. * @memberof DasNode */ @@ -529,4 +496,4 @@ DAS_API void del_RootNode(DasNode* pNode); } #endif -#endif /* _das_catalog_h_ */ +#endif /* _das_node_h_ */ diff --git a/das3/node_priv.h b/das3/node_priv.h new file mode 100644 index 0000000..2a5386f --- /dev/null +++ b/das3/node_priv.h @@ -0,0 +1,233 @@ +/* Copyright (C) 2026 Chris Piker + * + * This file is part of das2C, the Core Das2 C Library. + * + * Das2C is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License version 2.1 as published + * by the Free Software Foundation. + * + * Das2C is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for + * more details. + * + * You should have received a copy of the GNU Lesser General Public License + * version 2.1 along with das2C; if not, see . + */ + +/* Node internals shared by node.c and catalog.c. Not installed. + * + * DasNode is opaque to applications so that the layout below can change + * after the v3.0 tag without breaking them. C has no package-private, so + * the two files that need the fields include this header and everything + * else goes through the accessors in node.h. Functions declared here have + * no DAS_API: the library is built with hidden visibility, so they are + * reachable inside libdas3 and nowhere else. + */ + +#ifndef _das_node_priv_h_ +#define _das_node_priv_h_ + +#include "node.h" + +/* How a node tree works, for readers of node.c and catalog.c. + * + * Derived types. struct das_node is the common header of every node. + * Nodes that have children (Catalog, Collection, SourceSet and the SPASE + * and SPDF trees) are really a DasCatNode, defined in node.c, whose first + * member is a DasNode so that either can be handled through a DasNode*. + * The type is checked before casting, see DasNode_isCatalog(). Terminal + * nodes (stream sources, file aggregations) are a bare struct das_node. + * + * Loading. Every node begins as a JSON document that _DasNode_mkNode() + * reads from a file, from the cache, or over HTTP, and _DasNode_fromJson() + * turns into a struct of the right type. The parsed DasJdo is kept in pDom + * for the life of the node and the accessors read from it rather than + * copying fields out. DasNode_subNode() matches a relative path against + * the names in the parent's container dictionary, loads the matching + * child's URL and remembers the child in the parent's pSubNodes array, so + * a second lookup of the same path costs nothing. + * + * Ownership. Trees are owned from the top. A node with bIsRoot set is a + * memory root and del_RootNode() on it frees everything loaded below it. + * Callers never free a non-root node, it belongs to its parent's pSubNodes + * array. A memory root need not be a catalog root: _DasNode_newRoot() + * walks down from the federated roots to the requested node, marks that + * one as a root, then deletes the nodes above it (del_NonRootNode() skips + * roots). An application holding site:/uiowa/juno/wav thus owns a one + * node tree with no parent in memory. + * + * Inherited settings. pMgr, sAgent and pCache are borrowed from whoever + * made the root and copied into each child as it loads, so a DasCat's + * credentials, agent string and cache follow a whole walk without being + * passed at every call. None are owned by the node; the caller keeps them + * alive for as long as the tree exists. + */ + +/* Keeps track of downloaded nodes and their age. + * + * A manager object such as DasCat owns node caches, DasNode just uses the cache. + */ +typedef struct das_node_cache { + char sDir[256]; /* empty means no caching */ + int nMaxAgeSec; /* zero or less means always fetch, but still save */ +} DasNodeCache; + +struct das_node { + das_node_type_e nType; + char sURL[512]; /* where this node was loaded from */ + char sPath[512]; /* the node's catalog path URI, may be empty */ + bool bIsRoot; /* a memory root, owns everything loaded below it */ + void* pDom; /* the parsed document, a DasJdo for JSON nodes */ + DasCredMngr* pMgr; /* borrowed from the loader, inherited by children */ + const char* sAgent; /* borrowed, usually a string literal */ + const DasNodeCache* pCache; /* borrowed, inherited by children */ +}; + +/* Root constructors. The public new_RootNode() and new_RootNode_url() in + * node.h are thin wrappers that pass NULL for pCache and so use the process + * default from DasNode_setCache(). A DasCat passes its own settings. */ + +/* Load the node at a catalog path URI by walking down from the built in + * federated roots, see das_root_urls(). + * + * Each root URL is tried in turn with a short connect timeout and the whole + * list is retried with longer ones before giving up, so one slow mirror + * does not stall a walk. The node found is marked as a memory root and + * every node above it is freed, see Ownership above. Credentials are not + * offered to the top level roots, only to the nodes below them, which is + * where restricted catalogs live. + * + * @param sPathUri The full catalog URI of the node wanted, for example + * "tag:das2.org,2012:site:/uiowa/juno/wav". The "site:" and "test:" + * shorthands are expanded to full tag URIs first. NULL returns the + * federated root node itself. + * + * @param pMgr The credentials manager offered to nodes below the top level, + * and inherited by every node loaded later from the result. May be + * NULL if no restricted catalogs will be walked. Borrowed. + * + * @param sAgent The HTTP user agent string sent with every request, also + * inherited. Borrowed, usually a string literal. + * + * @param pCache Where fetched node documents are saved and how long they + * are trusted, inherited by the whole tree. NULL selects the process + * default from DasNode_setCache(). Borrowed. + * + * @returns A new memory root, to be freed with del_RootNode(), or NULL if no + * federated root could be reached or none of them lead to sPathUri. + * Failures are logged, not raised through das_error(). + */ +DasNode* _DasNode_newRoot( + const char* sPathUri, DasCredMngr* pMgr, const char* sAgent, + const DasNodeCache* pCache +); + +/* Load one node directly from a URL or file path and make it a memory root. + * + * The maximum connect timeout is used since there is no alternate URL to + * fall back on. + * + * @param sUrl An http://, https:// or file:// URL, or a bare file system + * path, of a node document. Local files are read directly and never + * cached. + * + * @param sPathUri The catalog URI to record for the node, which the loader + * has no way to know from the URL alone. It may be any name the + * caller likes, or "" when no URI applies, as for a nameless DasCat + * root. Children extend it using the node's own separator. + * + * @param pMgr Credentials for this fetch and for every node loaded later + * from the result. May be NULL. Borrowed. + * + * @param sAgent The HTTP user agent string, inherited by children. + * Borrowed, usually a string literal. + * + * @param pCache Cache settings for the tree, NULL for the process default. + * Borrowed. + * + * @returns A new memory root, to be freed with del_RootNode(), or NULL if + * the document could not be read or is not a recognized node type. + * Failures are logged, not raised through das_error(). + */ +DasNode* _DasNode_newRootUrl( + const char* sUrl, const char* sPathUri, DasCredMngr* pMgr, const char* sAgent, + const DasNodeCache* pCache +); + +/* A synthetic Catalog node whose children are other nodes placed at chosen + * catalog URIs. This is the machinery behind DasCat_mount() and the named + * form of DasCat_setRoot(). + * + * It works the way the federated root catalog at das2.org does. That node + * has "separator":null, so its children are named by whole URI prefixes + * such as "tag:das2.org,2012:" and node.c cannot split a path on a + * separator to find one. Instead _DasNode_subNode() walks the children in + * document order and takes the FIRST child whose name is a prefix of the + * requested path, then resolves the rest of the path below that child. + * + * A mount root is a small JSON document of that shape, written in memory + * and parsed like any downloaded node. Each child is named by the full URI + * of a mount and its urls list holds the file or URL to load there. For + * the two mounts (./uiowa.json at site:/uiowa) and (./wav.json at + * site:/uiowa/juno/wav) with fall-through, the document is in spirit: + * + * { "type":"Catalog", "separator":null, "catalog":{ + * "tag:das2.org,2012:site:/uiowa/juno/wav": {"urls":["./wav.json"]}, + * "tag:das2.org,2012:site:/uiowa": {"urls":["./uiowa.json"]}, + * "": {"urls":["https://das2.org/catalog/index.json", ...]} + * }} + * + * Resolving site:/uiowa/juno/wav/survey matches the wav child, loads + * wav.json and looks up "/survey" inside it. site:/uiowa/cassini skips + * wav and matches uiowa. A CDAWeb URI matches neither mount, but the + * empty string is a prefix of every path, so the last child wins and the + * federated roots take over. + * + * Ordering matters because the first match wins, not the best match. If + * uiowa were listed before wav it would also claim every wav path and + * wav.json would never be consulted. Hence the children are written + * longest URI first, so a more specific mount shadows a broader one. + * + * With bFallThrough the empty-named child is appended and any path no mount + * claims resolves through the normal catalog. Without it the mounts are + * the whole catalog and everything else fails to resolve. A named root is + * exactly that with a single mount, which is how DasCat_setRoot() builds + * one. + * + * @param aUrl The URL or file path to load for each mount, in the same order + * as aUri. None may be NULL or empty. + * + * @param aUri The full catalog URI each mount sits at, for example + * "tag:das2.org,2012:site:/uiowa". None may be NULL or empty, and + * the "site:" shorthand is NOT expanded here, callers such as DasCat + * do that first. Order does not matter, the children are sorted. + * + * @param uMounts The number of entries in aUrl and aUri, at least 1 and at + * most 64, the size of the sort buffer in node.c. + * + * @param bFallThrough If true, paths no mount claims resolve through the + * built in federated roots. If false the mounts are the whole + * catalog. + * + * @param pMgr Credentials offered when the mounted nodes and their children + * are loaded. May be NULL. Borrowed. + * + * @param sAgent The HTTP user agent string, inherited by children. + * Borrowed, usually a string literal. + * + * @param pCache Cache settings for the tree, NULL for the process default. + * Borrowed. + * + * @returns A new memory root whose source URL reports as "memory:mounts", + * to be freed with del_RootNode(), or NULL if the arguments are bad + * or the document could not be allocated. Nothing is fetched at + * this point; the mounted nodes load on first lookup. + */ +DasNode* _DasNode_newMountRoot( + const char** aUrl, const char** aUri, size_t uMounts, + bool bFallThrough, DasCredMngr* pMgr, const char* sAgent, + const DasNodeCache* pCache +); + +#endif /* _das_node_priv_h_ */ diff --git a/test/TestCat.c b/test/TestCat.c new file mode 100644 index 0000000..f37138a --- /dev/null +++ b/test/TestCat.c @@ -0,0 +1,309 @@ +/** @file TestCat.c Unit tests for DasCat, the catalog session, over local + * node files. No network. */ + +/* Author: Chris Piker , via Claude Fable 5.1 + * + * This file is intended to demonstrate an interface. This is free + * and unencumbered software released into the public domain + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this file, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this file dedicate any and all copyright interest in this file to + * the public domain. We make this dedication for the benefit of the + * public at large and to the detriment of our heirs and successors. We + * intend this dedication to be an overt act of relinquishment in + * perpetuity of all present and future rights to this file under + * copyright law. + * + * THIS FILE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * + * For more information, please refer to + */ + +/* Runs from the top of the source tree, the fixture node files in + * test/das_fetch/cat/ refer to each other by paths from there. The tree + * is site -> uiowa -> geomag -> flex, with two children under uiowa that + * can be listed but not loaded. */ + +#define _POSIX_C_SOURCE 200112L + +#include +#include +#include +#include + +#include +#include + +#define SITE "test/das_fetch/cat/site.json" +#define UIOWA "test/das_fetch/cat/uiowa.json" +#define UIOWA_URI "tag:das2.org,2012:site:/uiowa" + +static int g_nFail = 0; +static int g_nPass = 0; + +#define CHECK(cond, ...) do{ \ + if(cond){ ++g_nPass; } \ + else{ ++g_nFail; printf(" FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } \ +}while(0) + +static bool streq(const char* a, const char* b) +{ + if((a == NULL)||(b == NULL)) return (a == b); + return strcmp(a, b) == 0; +} + +/* Refusals log at error level by design; keep expected ones out of the + * test output */ +static DasErrCode quiet(DasErrCode (*f)(DasCat*, const char*), DasCat* p, const char* s) +{ + daslog_setlevel(DASLOG_CRIT); + DasErrCode n = f(p, s); + daslog_setlevel(DASLOG_WARN); + return n; +} +/* for the two-string setters, argument order is (url, uri) as in catalog.h */ +static DasErrCode quiet2( + DasErrCode (*f)(DasCat*, const char*, const char*), DasCat* p, const char* a, const char* b +){ + daslog_setlevel(DASLOG_CRIT); + DasErrCode n = f(p, a, b); + daslog_setlevel(DASLOG_WARN); + return n; +} +static DasNode* quietResolve(DasCat* p, const char* s) +{ + daslog_setlevel(DASLOG_CRIT); + DasNode* pNode = DasCat_resolve(p, s); + daslog_setlevel(DASLOG_WARN); + return pNode; +} + +/* ************************************************************************* */ +static void test_isUrlOrFile(void) +{ + printf("TEST: das_isUrlOrFile\n"); + CHECK(das_isUrlOrFile("http://a/b.json"), "http"); + CHECK(das_isUrlOrFile("https://a/b"), "https"); + CHECK(das_isUrlOrFile("file:///x/y"), "file scheme"); + CHECK(das_isUrlOrFile("./node.json"), "dot path"); + CHECK(das_isUrlOrFile("/abs/node"), "absolute path"); + CHECK(das_isUrlOrFile("C:\\nodes\\x"), "windows drive path by backslash"); + CHECK(das_isUrlOrFile("test/das_fetch/cat/site.json"), "relative path ending in .json"); + CHECK(!das_isUrlOrFile("uiowa/juno/wav/survey"), "catalog path"); + CHECK(!das_isUrlOrFile("tag:das2.org,2012:site:/uiowa"), "catalog URI"); + CHECK(!das_isUrlOrFile("site:/uiowa"), "shorthand URI"); + CHECK(!das_isUrlOrFile(NULL), "NULL"); + CHECK(!das_isUrlOrFile(""), "empty"); +} + +/* ************************************************************************* */ +static void test_policy(void) +{ + printf("TEST: root policy setters\n"); + DasCat* pCat = new_DasCat(NULL, "TestCat"); + CHECK(pCat != NULL, "new_DasCat"); + CHECK(DasCat_credMngr(pCat) == NULL, "no manager"); + CHECK(streq(DasCat_agent(pCat), "TestCat"), "agent copied"); + CHECK(streq(DasCat_prefix(pCat), DASCAT_DEF_PREFIX), "default prefix"); + CHECK(!DasCat_hasRoot(pCat), "no root yet"); + CHECK(DasCat_rootUrl(pCat) == NULL, "no root url"); + CHECK(DasCat_rootUri(pCat) == NULL, "no root URI"); + CHECK(DasCat_mountCount(pCat) == 0, "no mounts"); + CHECK(DasCat_lastUri(pCat) == NULL, "no resolve yet"); + + CHECK(DasCat_setPrefix(pCat, "site:/uiowa") == DAS_OKAY, "prefix shorthand"); + CHECK(streq(DasCat_prefix(pCat), UIOWA_URI), "prefix expanded: %s", DasCat_prefix(pCat)); + CHECK(DasCat_setPrefix(pCat, "tag:das2.org,2012:test:/") == DAS_OKAY, "prefix full"); + CHECK(quiet(DasCat_setPrefix, pCat, SITE) != DAS_OKAY, "a file is not a prefix"); + CHECK(quiet(DasCat_setPrefix, pCat, "") != DAS_OKAY, "empty prefix"); + + CHECK(quiet2(DasCat_setRoot, pCat, "uiowa", NULL) != DAS_OKAY, "a path is not a root"); + CHECK(DasCat_setRoot(pCat, SITE, NULL) == DAS_OKAY, "root by file"); + CHECK(DasCat_hasRoot(pCat), "has root"); + CHECK(streq(DasCat_rootUrl(pCat), SITE), "root url kept"); + CHECK(quiet2(DasCat_setRoot, pCat, UIOWA, NULL) != DAS_OKAY, "one root only"); + CHECK(quiet2(DasCat_setRoot, pCat, UIOWA, UIOWA_URI) != DAS_OKAY, "no named root over a root"); + CHECK(quiet2(DasCat_mount, pCat, UIOWA, UIOWA_URI) != DAS_OKAY, "no mount over a root"); + CHECK(DasCat_rootUri(pCat) == NULL, "nameless root has no URI"); + del_DasCat(pCat); + + pCat = new_DasCat(NULL, NULL); + CHECK(DasCat_agent(pCat) == NULL, "NULL agent stays NULL"); + CHECK(quiet2(DasCat_mount, pCat, UIOWA, "") != DAS_OKAY, "a mount needs a URI"); + CHECK(quiet2(DasCat_setRoot, pCat, UIOWA, UIOWA) != DAS_OKAY, "root URI can't be a file"); + CHECK(quiet2(DasCat_setRoot, pCat, "uiowa", UIOWA_URI) != DAS_OKAY, "root URL must be a location"); + CHECK(DasCat_setRoot(pCat, UIOWA, "site:/uiowa") == DAS_OKAY, "named root"); + CHECK(streq(DasCat_rootUri(pCat), UIOWA_URI), "root URI expanded"); + CHECK(DasCat_hasRoot(pCat), "a named root is a root"); + CHECK(quiet2(DasCat_mount, pCat, UIOWA, UIOWA_URI) != DAS_OKAY, "no mount over a named root"); + CHECK(quiet2(DasCat_setRoot, pCat, SITE, NULL) != DAS_OKAY, "no second root"); + del_DasCat(pCat); + + pCat = new_DasCat(NULL, NULL); + CHECK(DasCat_mount(pCat, UIOWA, UIOWA_URI) == DAS_OKAY, "mount one"); + CHECK(DasCat_mount(pCat, SITE, "site:/swri") == DAS_OKAY, "mount two"); + CHECK(DasCat_mountCount(pCat) == 2, "two mounts"); + CHECK(!DasCat_hasRoot(pCat), "mounts are not a root"); + CHECK(quiet2(DasCat_setRoot, pCat, UIOWA, UIOWA_URI) != DAS_OKAY, "no named root over mounts"); + CHECK(quiet2(DasCat_setRoot, pCat, SITE, NULL) != DAS_OKAY, "no root over mounts"); + del_DasCat(pCat); +} + +/* ************************************************************************* */ +static void test_rootMode(void) +{ + printf("TEST: a node file as the only root\n"); + DasCat* pCat = new_DasCat(NULL, "TestCat"); + DasCat_setCache(pCat, NULL, 0); /* explicitly off, file nodes never cache anyway */ + CHECK(DasCat_setRoot(pCat, SITE, NULL) == DAS_OKAY, "set root"); + + DasNode* pRoot = DasCat_resolve(pCat, NULL); + CHECK(pRoot != NULL, "root resolves"); + CHECK(pRoot && DasNode_isCatalog(pRoot), "root is a catalog"); + CHECK(streq(DasCat_lastUri(pCat), SITE), "no path: last URI is the root location"); + CHECK(DasCat_resolve(pCat, "") == pRoot, "empty path is the same as NULL"); + + /* Policy is frozen now */ + CHECK(quiet(DasCat_setPrefix, pCat, "site:/") != DAS_OKAY, "no policy change after resolve"); + + DasNode* pSrc = DasCat_resolve(pCat, "uiowa/geomag/flex"); + CHECK(pSrc != NULL, "deep path resolves"); + CHECK(pSrc && DasNode_isStreamSrc(pSrc), "leaf is a stream source"); + CHECK(streq(DasCat_lastUri(pCat), "uiowa/geomag/flex"), "relative path is the URI under a URL root"); + CHECK(pSrc && streq(DasNode_pathUri(pSrc), "/uiowa/geomag/flex"), "node path URI: %s", + pSrc ? DasNode_pathUri(pSrc) : "(null)"); + CHECK(DasCat_resolve(pCat, "uiowa/geomag/flex") == pSrc, "second resolve is the cached node"); + + DasNode* pUiowa = DasCat_resolve(pCat, "uiowa"); + CHECK(pUiowa && DasNode_isCatalog(pUiowa), "intermediate node"); + CHECK(pUiowa && (DasNode_subCount(pUiowa) == 3), "uiowa lists three children"); + CHECK(pUiowa && (DasNode_subType(pUiowa, 2) == d2node_inv), "unknown child type is d2node_inv"); + + /* Misses, and what the catalog can say about them */ + CHECK(quietResolve(pCat, "uiowa/nope/deeper") == NULL, "bad child fails"); + CHECK(streq(DasCat_lastUri(pCat), "uiowa/nope/deeper"), "last URI is the failed path"); + CHECK(DasCat_covers(pCat, DasCat_lastUri(pCat)), "a URL root covers everything"); + const char* sRest = NULL; + DasNode* pNear = DasCat_nearest(pCat, &sRest); + CHECK(pNear == pUiowa, "nearest is the deepest catalog that loaded"); + CHECK(streq(sRest, "nope/deeper"), "rest below it: %s", sRest ? sRest : "(null)"); + CHECK(streq(DasCat_lastUri(pCat), "uiowa"), "last URI now names the nearest node"); + + CHECK(quietResolve(pCat, "uiowa/broken") == NULL, "listed child with a missing file fails"); + pNear = DasCat_nearest(pCat, &sRest); + CHECK((pNear == pUiowa) && streq(sRest, "broken"), "nearest for a missing file"); + + CHECK(quietResolve(pCat, "uiowa/odd") == NULL, "unknown node type fails to load"); + + CHECK(quietResolve(pCat, "nope") == NULL, "bad child of the root"); + pNear = DasCat_nearest(pCat, &sRest); + CHECK(pNear == NULL, "nothing above a root child"); + CHECK(streq(DasCat_lastUri(pCat), "nope"), "a failed nearest keeps the failed URI"); + + /* A good resolve leaves nothing for nearest to find */ + CHECK(DasCat_resolve(pCat, "uiowa/geomag") != NULL, "source set resolves"); + CHECK(DasCat_nearest(pCat, &sRest) == pUiowa, "nearest after success is the parent, by design"); + + /* A location given as the path, named by itself */ + DasNode* pDirect = DasCat_resolve(pCat, UIOWA); + CHECK(pDirect && DasNode_isCatalog(pDirect), "node by file path"); + CHECK(streq(DasCat_lastUri(pCat), UIOWA), "location is its own URI"); + CHECK(pDirect && streq(DasNode_pathUri(pDirect), UIOWA), "node path URI is the location"); + CHECK(DasCat_nearest(pCat, &sRest) == NULL, "nearest means nothing for a location"); + + del_DasCat(pCat); +} + +/* ************************************************************************* */ +static void test_namedRoot(void) +{ + printf("TEST: a named root\n"); + DasCat* pCat = new_DasCat(NULL, "TestCat"); + CHECK(DasCat_setRoot(pCat, UIOWA, "site:/uiowa") == DAS_OKAY, "named root"); + + DasNode* pTop = DasCat_resolve(pCat, NULL); + CHECK(pTop && DasNode_isCatalog(pTop), "no path: the named node"); + CHECK(streq(DasCat_lastUri(pCat), UIOWA_URI), "its URI"); + CHECK(pTop && streq(DasNode_pathUri(pTop), UIOWA_URI), "node knows its given URI"); + + DasNode* pSrc = DasCat_resolve(pCat, "geomag/flex"); + CHECK(pSrc && DasNode_isStreamSrc(pSrc), "relative to the root"); + CHECK(streq(DasCat_lastUri(pCat), UIOWA_URI "/geomag/flex"), "relative path gets the root URI: %s", + DasCat_lastUri(pCat)); + CHECK(pSrc && streq(DasNode_pathUri(pSrc), UIOWA_URI "/geomag/flex"), "node path URI"); + + CHECK(DasCat_resolve(pCat, UIOWA_URI "/geomag") != NULL, "full URI under the root"); + CHECK(DasCat_resolve(pCat, "site:/uiowa/geomag") != NULL, "shorthand URI under the root"); + CHECK(streq(DasCat_lastUri(pCat), UIOWA_URI "/geomag"), "shorthand expanded"); + + CHECK(quietResolve(pCat, "tag:das2.org,2012:site:/swri/x") == NULL, "outside the root fails"); + CHECK(!DasCat_covers(pCat, DasCat_lastUri(pCat)), "and the policy says so"); + CHECK(DasCat_covers(pCat, UIOWA_URI "/anything"), "under the root is covered"); + + CHECK(quietResolve(pCat, "geomag/nope") == NULL, "bad child under the root"); + const char* sRest = NULL; + DasNode* pNear = DasCat_nearest(pCat, &sRest); + CHECK(pNear && DasNode_isCatalog(pNear) && streq(sRest, "nope"), "nearest under a named root: %s", + sRest ? sRest : "(null)"); + CHECK(streq(DasCat_lastUri(pCat), UIOWA_URI "/geomag"), "nearest URI under a named root"); + + del_DasCat(pCat); +} + +/* ************************************************************************* */ +static void test_mounts(void) +{ + printf("TEST: mounts on the federated catalog (mounted side only, offline)\n"); + DasCat* pCat = new_DasCat(NULL, "TestCat"); + CHECK(DasCat_mount(pCat, UIOWA, "site:/uiowa") == DAS_OKAY, "mount"); + + /* Without a prefix relative paths start at the first mount */ + DasNode* pSrc = DasCat_resolve(pCat, "geomag/flex"); + CHECK(pSrc && DasNode_isStreamSrc(pSrc), "relative to the mount"); + CHECK(streq(DasCat_lastUri(pCat), UIOWA_URI "/geomag/flex"), "URI under the mount"); + + /* Full URIs under the mount never touch the network */ + CHECK(DasCat_resolve(pCat, "site:/uiowa/geomag") != NULL, "full URI through the mount"); + CHECK(DasCat_covers(pCat, "tag:das2.org,2012:site:/swri"), "a mount leaves the rest of the catalog reachable"); + del_DasCat(pCat); + + /* With a prefix relative paths start there, and land on the mount */ + pCat = new_DasCat(NULL, "TestCat"); + CHECK(DasCat_setPrefix(pCat, "site:/") == DAS_OKAY, "prefix"); + CHECK(DasCat_mount(pCat, UIOWA, "site:/uiowa") == DAS_OKAY, "mount after prefix"); + pSrc = DasCat_resolve(pCat, "uiowa/geomag/flex"); + CHECK(pSrc && DasNode_isStreamSrc(pSrc), "prefix + path through the mount"); + CHECK(streq(DasCat_lastUri(pCat), UIOWA_URI "/geomag/flex"), "URI: %s", DasCat_lastUri(pCat)); + del_DasCat(pCat); + + /* Two mounts, the longer URI must win when one is under the other */ + pCat = new_DasCat(NULL, "TestCat"); + CHECK(DasCat_mount(pCat, SITE, "site:/uiowa") == DAS_OKAY, "wide mount of the wrong file"); + CHECK(DasCat_mount(pCat, "test/das_fetch/cat/geomag.json", "site:/uiowa/geomag") == DAS_OKAY, + "narrow mount"); + DasNode* pNode = DasCat_resolve(pCat, "site:/uiowa/geomag/flex"); + CHECK(pNode && DasNode_isStreamSrc(pNode), "narrow mount shadows the wide one"); + del_DasCat(pCat); +} + +/* ************************************************************************* */ +int main(int argc, char** argv) +{ + das_init(argv[0], DASERR_DIS_EXIT, 0, DASLOG_WARN, NULL); + + test_isUrlOrFile(); + test_policy(); + test_rootMode(); + test_namedRoot(); + test_mounts(); + + printf("%d checks passed, %d failed\n", g_nPass, g_nFail); + return (g_nFail == 0) ? 0 : 1; +} diff --git a/test/TestCatalog.c b/test/TestCatalog.c index 1fb057f..fa9be9a 100644 --- a/test/TestCatalog.c +++ b/test/TestCatalog.c @@ -99,7 +99,7 @@ int main(int argc, char** argv) { printf("TEST %d: (Not) Deleting a non-root ", nTest); del_RootNode((DasNode*)pSurvey); /* Erasing const for testing purposes */ /* Shouldn't be done in app code */ - if(pSurvey->sURL[0] != '\0') printf(" ...looks like it's still here [OKAY]\n"); + if(DasNode_srcUrl(pSurvey)[0] != '\0') printf(" ...looks like it's still here [OKAY]\n"); else printf("... [FAILED]"); nTest += 1; @@ -117,7 +117,7 @@ int main(int argc, char** argv) { return nTest; } else{ - printf("TEST %d: Root node loaded from %s [OKAY]\n", nTest, pRoot->sURL); + printf("TEST %d: Root node loaded from %s [OKAY]\n", nTest, DasNode_srcUrl(pRoot)); } nTest += 1; diff --git a/test/das_fetch/cat/geomag.json b/test/das_fetch/cat/geomag.json new file mode 100644 index 0000000..6e129fe --- /dev/null +++ b/test/das_fetch/cat/geomag.json @@ -0,0 +1,14 @@ +{ + "type": "SourceSet", + "version": "0.5", + "name": "geomag", + "title": "Geomagnetic coordinates, offline test copy", + "separator": "/", + "catalog": { + "flex": { + "type": "HttpStreamSrc", + "title": "dasflex server", + "urls": ["test/das_fetch/geomag_flex_0.7.json"] + } + } +} diff --git a/test/das_fetch/cat/site.json b/test/das_fetch/cat/site.json new file mode 100644 index 0000000..503247e --- /dev/null +++ b/test/das_fetch/cat/site.json @@ -0,0 +1,14 @@ +{ + "type": "Catalog", + "version": "0.5", + "name": "site", + "title": "Offline test catalog, stands in for tag:das2.org,2012:site:", + "separator": "/", + "catalog": { + "uiowa": { + "type": "Catalog", + "title": "University of Iowa", + "urls": ["test/das_fetch/cat/uiowa.json"] + } + } +} diff --git a/test/das_fetch/cat/uiowa.json b/test/das_fetch/cat/uiowa.json new file mode 100644 index 0000000..105e28c --- /dev/null +++ b/test/das_fetch/cat/uiowa.json @@ -0,0 +1,24 @@ +{ + "type": "Catalog", + "version": "0.5", + "name": "uiowa", + "title": "University of Iowa, offline test copy", + "separator": "/", + "catalog": { + "geomag": { + "type": "SourceSet", + "title": "Geomagnetic coordinates", + "urls": ["test/das_fetch/cat/geomag.json"] + }, + "broken": { + "type": "Catalog", + "title": "Listed, but its file does not exist", + "urls": ["test/das_fetch/cat/missing.json"] + }, + "odd": { + "type": "WebSocSrc", + "title": "A node type this library does not load", + "urls": ["test/das_fetch/cat/odd.json"] + } + } +} diff --git a/test/das_fetch_test.sh b/test/das_fetch_test.sh index c9c13bc..b2e8ff1 100755 --- a/test/das_fetch_test.sh +++ b/test/das_fetch_test.sh @@ -97,7 +97,7 @@ expect_url $G -- \ expect_url -r $G 0 -- \ "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" -# a seed, URI=URL: the node at URL is named URI, relative paths hang off it +# a named root, URI=URL: the node at URL is named URI, relative paths hang off it SU="tag:das2.org,2012:site:/uiowa/tracers/tracers-2/spice/geomagcoords/flex" expect_url -r $SU=$G 0 -- \ "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" @@ -106,21 +106,21 @@ expect_url -r $SU=$G $SU 0 -- \ if $FETCH node -r $SU=$G 2>/dev/null | grep -q "^URI: $SU\$"; then nPass=$((nPass + 1)) else - echo " FAIL: das_fetch node -r $SU=$G did not report the seed URI"; nFail=$((nFail + 1)) + echo " FAIL: das_fetch node -r $SU=$G did not report the root URI"; nFail=$((nFail + 1)) fi -# -g grafts the same seed over the federated catalog. Offline only the -# seed side can be exercised; -g and -r URL together is a usage error. -expect_url -g $SU=$G 0 -- \ +# -m mounts the same node on the federated catalog. Offline only the +# mounted side can be exercised; -m and -r URL together is a usage error. +expect_url -m $SU=$G 0 -- \ "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" -export DASFETCH_GRAFT=$SU=$G +export DASFETCH_MOUNT=$SU=$G expect_url 0 -- \ "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" -unset DASFETCH_GRAFT -expect_rc 2 url -g $SU=$G -r $H 0 # two roots -expect_rc 2 url -g site:/uiowa 0 # graft needs URI=URL +unset DASFETCH_MOUNT +expect_rc 2 url -m $SU=$G -r $H 0 # a root and a mount +expect_rc 2 url -m site:/uiowa 0 # mount needs URI=URL expect_rc 2 url -r site:/uiowa=nourl 0 # '=' but no URL after it -expect_rc 3 url -r $SU=$G site:/uiowa/juno # -r seed is the whole catalog +expect_rc 3 url -r $SU=$G site:/uiowa/juno # a named root is the whole catalog # DASFETCH_ROOT stands in for -r, and -r still wins over it export DASFETCH_ROOT=$G diff --git a/utilities/das3_node.c b/utilities/das3_node.c deleted file mode 100644 index c644e20..0000000 --- a/utilities/das3_node.c +++ /dev/null @@ -1,212 +0,0 @@ -/* This small utility is mostly meant to demonstrate an interface and thus - * has a liberal open source license... - * - * The MIT License - * - * Copyright 2023 Chris Piker - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#define _POSIX_C_SOURCE 200112L - -#include -#include - -#include -#include -#include - -#define PROG_ERR 64 - -#define AGENT "das2C" - -/* ************************************************************************* */ -void prnHelp() -{ - fprintf(stderr, -"SYNOPSIS\n" -" das_node - Read a node out of the federated catalog\n" -"\n" -"USAGE\n" -" das_node [-h] [-R] [-a ALT_ROOT] [TAG_URI [#FRAGMENT] ]\n" -"\n" -"DESCRIPTION\n" -" das_node is a small utility which resolves a catalog TAG_URI to URL and then\n" -" writes the named catalog node to standard output. By default the\n" -" builtin root nodes are loaded first, then the catalog is walked to find the\n" -" requested sub-node. The walking algorithm automatically backs-up and tries\n" -" alternate branches when URL resolution fails. If there is only one URL for\n" -" a given branch, or if walking all branches still fails to load the node then\n" -" resolution fails.\n" -"\n" -" The TAG_URI may include a document fragment by including the hash symbol and\n" -" then the relative path of the object within the document.\n Paths are clearly\n" -" defined for JSON nodes, but not necessarily for other catalog object formats.\n" -"\n" -" Any node of type Catalog may be a used as the root node. To do so provide\n" -" and absolute URL to the root using '-a' below.\n" -"\n" -"OPTIONS\n" -"\n" -" -h,--help\n" -" Print this help text\n" -"\n" -" -R,--roots\n" -" Print the builtin root URLs and exit\n" -"\n" -" -a URL,--alt-root URL\n" -" Don't use the compiled in root URLs, look for the given object under\n" -" this alternate root catalog object. Useful for testing detached\n" -" catalogs.\n" -"\n" -" -l,--level\n" -" The logging level, one of 'none','crit', 'error', 'warn', 'info', \n" -" 'debug', or 'trace'.\n" -"\n" -"\n" -"EXAMPLES\n" -" Print the compiled in default federated catalog roots:\n" -" das_node -R\n" -"\n" -" Get the U. Iowa Juno site data source catalog:\n" -" das_node tag:das2.org,2012:site:/uiowa/juno\n" -"\n" -" Retrieve a HttpStreamSrc node for Juno Waves Survey data given an\n" -" explicit URL for the root node:\n" -" das_node -a https://das2.org/catalog/das/site/uiowa.json juno/wav/survey/das2\n" -"\n" -" Output only the protocol section of a source catalog object:\n" -" das_node tag:das2.org,2012:site:/uiowa/juno/survey/das2#protocol\n" -"\n" -"AUTHOR\n" -" chris-piker@uiowa.edu\n" -"\n"); -} - -/* ************************************************************************* */ - -int main(int argc, char** argv) -{ - /* Exit on errors, log info messages and above */ - das_init(argv[0], DASERR_DIS_EXIT, 0, DASLOG_INFO, NULL); - - const char* sRootUrl = NULL; - const char* sNodeUri = NULL; - const char* sNodeFrag = NULL; - size_t uRoots = 0; - const char** psRoots; - - for(int i = 1; i < argc; i++){ - if(strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0 ){ - prnHelp(); - return 0; - } - - if(strcmp(argv[i], "-R") == 0 || strcmp(argv[i], "--roots") == 0 ){ - psRoots = das_root_urls(&uRoots); - fprintf(stdout, "Compiled in das federated catalog URLs:\n"); - for(size_t u = 0; u < uRoots; ++u) - fprintf(stdout, " %s\n", psRoots[u]); - return 0; - } - - if(strcmp(argv[i], "-l") == 0 || strcmp(argv[i], "--level") == 0){ - if(argc <= i){ - daslog_error("Logging level argument missing, use -h for help"); - return 4; - } - ++i; - daslog_setlevel(daslog_strlevel(argv[i])); - continue; - } - - if(strcmp(argv[i], "-a") == 0 || strcmp(argv[i], "--alt-root") == 0){ - if(argc <= i){ - daslog_error( - "Alternate root URL missing after -a or --alt-root, use -h for help." - ); - return 5; - } - ++i; - sRootUrl = argv[i]; - continue; - } - - if(sNodeUri == NULL){ - sNodeUri = argv[i]; - char* pHash = strchr(sNodeUri, '#'); - if(pHash != NULL){ - *pHash = '\0'; - sNodeFrag = pHash + 1; - } - continue; - } - - daslog_error_v("Unknown extra parameter: %s\n", argv[i]); - return 3; - } - - DasNode* pRoot; - if(sRootUrl) - pRoot = new_RootNode_url(sRootUrl, NULL, NULL, AGENT); - else - pRoot = new_RootNode(NULL, NULL, AGENT); - - if(pRoot == NULL){ - fprintf(stderr, "ERROR: Couldn't get the root node"); - return 4; - } - - DasNode* pNode; - if(sNodeUri) - pNode = DasNode_subNode(pRoot, sNodeUri, NULL, AGENT); - else - pNode = pRoot; - - if(pNode != NULL){ - fprintf(stderr, - "Loaded node: %s\n" - "From URL: %s\n", - DasNode_name(pNode), DasNode_srcUrl(pNode) - ); - } - else{ - daslog_error_v("Couldn't load %s starting from %s", sNodeUri, DasNode_srcUrl(pRoot)); - return 7; - } - - - if(DasNode_isJson(pNode)){ - if(sNodeFrag != NULL) fprintf(stdout, "Fragment: %s\n", sNodeFrag); - const DasJdo* pJdo = DasNode_getJdo(pNode, sNodeFrag); - const char* pPretty = (char*) DasJdo_writePretty(pJdo, " ", "\n", NULL); - fprintf(stdout, "\n%s\n", pPretty); - } - else{ - fprintf(stdout, - "The object was type %d, there's no printer for it yet.\n", - pRoot->nType - ); - } - - del_RootNode(pRoot); - - return 0; -} diff --git a/utilities/das_fetch.c b/utilities/das_fetch.c index 0069f31..0f2965a 100644 --- a/utilities/das_fetch.c +++ b/utilities/das_fetch.c @@ -20,12 +20,13 @@ * server URL. * * The program has two halves. The small half is catalog navigation, which - * is almost entirely the library's job (das3/node.h). The large half is + * is the library's job (DasCat in das3/catalog.h, nodes in das3/node.h); only + * the command line spelling of the root policy lives here. The large half is * turning a data source's "interface" section into command line keys, and * turning the user's KEY=VALUE arguments back into HTTP GET parameters via - * the source's "protocol" section. That translation follows the same rules - * das2py uses in streamsrc.HttpStreamSrc._translate(), so the two clients - * agree on what a source definition means. + * the source's "protocol" section. The translation rules are those of + * the das catalog schema for HttpStreamSrc nodes, so any client following + * the schema agrees on what a source definition means. * * Two generations of HttpStreamSrc node are in the wild (0.6 from 2018, * 0.7 from dasflex) and they spell things differently. Layout detection @@ -50,6 +51,7 @@ #include #include +#include #include #ifdef _WIN32 @@ -65,9 +67,8 @@ #define PROG "das_fetch" #define PERR (DASERR_MAX + 1) -#define DEF_ROOT "tag:das2.org,2012:site:/" #define DAS_DIR ".das" /* under the home directory, holds cache/ and servers */ -#define CRED_FILE "servers" /* das2C leads here, das2py follows; ~/.dasauth is not read */ +#define CRED_FILE "servers" /* under DAS_DIR; the older ~/.dasauth is not read */ #ifndef _WIN32 #define HOME_VAR "$HOME" /* for help text, das_userhome() does the lookup */ @@ -100,7 +101,7 @@ #define DEF_CACHE_AGE 3600 /* seconds a cached catalog node is trusted */ #define ROOT_ENV "DASFETCH_ROOT" -#define GRAFT_ENV "DASFETCH_GRAFT" +#define MOUNT_ENV "DASFETCH_MOUNT" /* ************************************************************************* */ /* Help */ @@ -146,8 +147,8 @@ static void prnHelpMain(void) " Paths are catalog paths, not web paths. Any of these forms work:\n" "\n" " uiowa/juno/wav/survey relative to the root prefix, by\n" -" default '" DEF_ROOT "'\n" -" site:/uiowa/juno/wav/survey das2py shorthand\n" +" default '" DASCAT_DEF_PREFIX "'\n" +" site:/uiowa/juno/wav/survey shorthand for the URI form below\n" " tag:das2.org,2012:site:/uiowa full catalog path URI\n" " https://host/some/node.json a node by direct URL\n" " ./local_node.json a node file on disk\n" @@ -158,7 +159,7 @@ static void prnHelpMain(void) "\n" " -r, --root ROOT Where paths start, in one of three forms.\n" " A catalog URI prefix for relative paths, default\n" -" '" DEF_ROOT "':\n" +" '" DASCAT_DEF_PREFIX "':\n" " -r tag:das2.org,2012:test:/\n" " A URL or file of a catalog node. It becomes the only\n" " root, nothing outside it can be reached:\n" @@ -170,15 +171,15 @@ static void prnHelpMain(void) " -r 10.1037/a0040251=https://doi.org/10.1037/a0040251/root.json\n" " Without -r the environment variable " ROOT_ENV " is\n" " used the same way, if set.\n" -" -g, --graft URI=URL\n" -" Patch the federated catalog: the node at URL stands in\n" -" for URI and everything under it, the rest of the\n" -" catalog is unchanged. Relative paths start at the\n" -" grafted node, or at the -r prefix if one is given:\n" -" -g site:/uiowa/tracers/tracers-1=\\\n" +" -m, --mount URI=URL\n" +" Mount a node on the federated catalog: the node at URL\n" +" stands in for URI and everything under it, the rest\n" +" of the catalog is unchanged. Relative paths start at\n" +" the mounted node, or at the -r prefix if one is given:\n" +" -m site:/uiowa/tracers/tracers-1=\\\n" " https://tracers-dev.physics.uiowa.edu/stream/source/tracers-1.json\n" -" The environment variable " GRAFT_ENV " stands in\n" -" for -g, if set.\n" +" May be repeated. The environment variable " MOUNT_ENV "\n" +" stands in for -m, if set.\n" "\n" " -s SECTIONS For 'node', which parts of a source to print, a comma\n" " separated list from: examples, coords, data, formats,\n" @@ -385,8 +386,8 @@ static void prnHelpRoots(void) "\n" " Print the compiled in URLs of the federated catalog root, one per line.\n" " The root is mirrored, the first URL that answers is used. A -r ROOT\n" -" argument or the " ROOT_ENV " variable replaces it, a -g URI=URL argument\n" -" or the " GRAFT_ENV " variable patches it, see '" PROG " -h'.\n" +" argument or the " ROOT_ENV " variable replaces it, a -m URI=URL argument\n" +" or the " MOUNT_ENV " variable mounts a node on it, see '" PROG " -h'.\n" "\n", stdout); } @@ -415,11 +416,6 @@ static bool strieq(const char* a, const char* b) return (*a == '\0') && (*b == '\0'); } -static bool strpfx(const char* s, const char* sPrefix) -{ - return (s != NULL) && (strncmp(s, sPrefix, strlen(sPrefix)) == 0); -} - static void strcpyz(char* sDest, const char* sSrc, size_t uDest) { if(uDest == 0) return; @@ -481,8 +477,8 @@ static bool jscalar(const DasJdo* p, char* sBuf, size_t uLen) } /* Percent encode everything but RFC 3986 unreserved characters. Servers - * decode before use, so over-encoding ':' in time strings is harmless and - * matches what das2py's quote_plus does. */ + * decode before use, so over-encoding ':' in time strings is harmless, and + * it is what most URL encoders do anyway. */ static void urlEncode(const char* sIn, char* sOut, size_t uOut) { static const char* sHex = "0123456789ABCDEF"; @@ -1684,231 +1680,49 @@ static const char* nodeUri( /* ************************************************************************* */ /* Catalog navigation * - * DasCat is a pilot for a library object: a catalog session that owns its - * roots, its credentials, its agent and the memory of every node it loads, - * and resolves user paths under one root policy. Today new_RootNode() and - * DasNode_subNode() leave all of that to the caller, which is why this is - * here rather than in node.c; see cf_notes/api_enlargement.md. Only - * applyRootArg(), the command line spelling of the policy, would stay in - * das_fetch when the rest moves. */ - -#define MAX_ROOTS 8 - -typedef struct das_cat { - DasCredMngr* pMgr; /* borrowed, the caller keeps it alive */ - const char* sAgent; /* borrowed */ - - /* Root policy. Relative paths start at sPrefix in the federated - catalog, unless a root URL or a seed is set, then they start at that - node (or, for a graft that also has an explicit prefix, at the prefix). */ - char sPrefix[URI_SZ]; /* default DEF_ROOT */ - bool bPrefixGiven; /* set by the caller, not just the default */ - char sRootUrl[URL_SZ]; /* a node by URL or file as the only root */ - char sSeedUri[URI_SZ]; /* URI=URL: the node at sSeedUrl is named sSeedUri */ - char sSeedUrl[URL_SZ]; - bool bFallThrough; /* the seed patches the federated roots (a graft) */ - - DasNode* pRoot; /* root node of the URL and seed modes, made on first use */ - DasNode* aRoots[MAX_ROOTS]; /* every memory root made, freed by DasCat_free */ - size_t uRoots; - char sLastUri[URI_SZ]; /* the path URI as the last resolve understood it */ -} DasCat; - -static bool isDirectUrl(const char* sPath) -{ - if(sPath == NULL) return false; - if(strpfx(sPath, "http://")||strpfx(sPath, "https://")||strpfx(sPath, "file://")) return true; - if((sPath[0] == '.')||(sPath[0] == '/')||(sPath[0] == '\\')) return true; - size_t u = strlen(sPath); - return (u > 5) && (strcmp(sPath + u - 5, ".json") == 0); -} - -static void DasCat_init(DasCat* pCat, DasCredMngr* pMgr, const char* sAgent) -{ - memset(pCat, 0, sizeof(DasCat)); - pCat->pMgr = pMgr; - pCat->sAgent = sAgent; - strcpyz(pCat->sPrefix, DEF_ROOT, sizeof(pCat->sPrefix)); -} - -/* Frees every node loaded through this catalog. Node pointers handed out - * by DasCat_resolve() are invalid afterwards. */ -static void DasCat_free(DasCat* pCat) -{ - for(size_t u = 0; u < pCat->uRoots; ++u) del_RootNode(pCat->aRoots[u]); - pCat->uRoots = 0; - pCat->pRoot = NULL; -} - -/* Take ownership of a memory root */ -static DasNode* DasCat_keep(DasCat* pCat, DasNode* pRoot) -{ - if(pRoot == NULL) return NULL; - if(pCat->uRoots < MAX_ROOTS) pCat->aRoots[pCat->uRoots++] = pRoot; - else daslog_warn("Too many catalog roots in one run, leaking one"); - return pRoot; -} - -/* site:/x and test:/x are das2py shorthand for the two well known children - * of the das2 root. new_RootNode() expands them too; doing it here as well - * keeps the reported URI the full one. */ -static void DasCat_expand(const char* sIn, char* sOut, size_t uOut) -{ - if(strpfx(sIn, "site:")||strpfx(sIn, "test:")) snprintf(sOut, uOut, D2URI_ROOT "%s", sIn); - else strcpyz(sOut, sIn, uOut); -} - -static bool DasCat_hasRoot(const DasCat* pCat) -{ - return (pCat->sRootUrl[0] != '\0')||(pCat->sSeedUrl[0] != '\0'); -} - -/* The three policy setters return an error text, or NULL */ - -static const char* DasCat_setPrefix(DasCat* pCat, const char* sUri) -{ - if(isDirectUrl(sUri)) return "a path prefix is a catalog URI, not a URL or file"; - DasCat_expand(sUri, pCat->sPrefix, sizeof(pCat->sPrefix)); - pCat->bPrefixGiven = true; - return NULL; -} - -static const char* DasCat_setRoot(DasCat* pCat, const char* sUrl) -{ - if(DasCat_hasRoot(pCat)) return "a catalog has one root"; - if(!isDirectUrl(sUrl)) return "a root is a URL or a file"; - strcpyz(pCat->sRootUrl, sUrl, sizeof(pCat->sRootUrl)); - return NULL; -} - -/* The node at sUrl is named sUri. As the only root (bFallThrough false), - * or patched over the federated catalog (true), see new_RootNode_seed(). */ -static const char* DasCat_seed(DasCat* pCat, const char* sUri, const char* sUrl, bool bFallThrough) -{ - if(DasCat_hasRoot(pCat)) return "a catalog has one root"; - if((sUri == NULL)||(sUri[0] == '\0')) return "a seed needs a URI"; - if(!isDirectUrl(sUrl)) return "a seed needs a URL or file after the '='"; - DasCat_expand(sUri, pCat->sSeedUri, sizeof(pCat->sSeedUri)); - strcpyz(pCat->sSeedUrl, sUrl, sizeof(pCat->sSeedUrl)); - pCat->bFallThrough = bFallThrough; - return NULL; -} - -/* Root node of the URL and seed modes, loaded on first use */ -static DasNode* DasCat_root(DasCat* pCat) -{ - if(pCat->pRoot != NULL) return pCat->pRoot; - if(pCat->sSeedUrl[0] != '\0') - pCat->pRoot = new_RootNode_seed(pCat->sSeedUri, pCat->sSeedUrl, pCat->bFallThrough, - pCat->pMgr, pCat->sAgent); - else if(pCat->sRootUrl[0] != '\0') - pCat->pRoot = new_RootNode_url(pCat->sRootUrl, "", pCat->pMgr, pCat->sAgent); - return DasCat_keep(pCat, pCat->pRoot); -} - -/* Resolve a user path to a node under the root policy. The node is owned - * by the catalog. DasCat_lastUri() afterwards gives the path URI as it was - * understood, resolved or not, for messages. */ -static DasNode* DasCat_resolve(DasCat* pCat, const char* sPath) -{ - char* sUri = pCat->sLastUri; - size_t uLen = sizeof(pCat->sLastUri); - bool bNoPath = (sPath == NULL)||(sPath[0] == '\0'); - - /* A node by URL or file, named by itself */ - if(!bNoPath && isDirectUrl(sPath)){ - strcpyz(sUri, sPath, uLen); - return DasCat_keep(pCat, new_RootNode_url(sPath, sPath, pCat->pMgr, pCat->sAgent)); - } - - if(pCat->sSeedUrl[0] != '\0'){ - DasNode* pRoot = DasCat_root(pCat); - if(pRoot == NULL) return NULL; - - /* No path: the seeded node itself. A full URI, or a relative path - under an explicit prefix when this is a graft: the seed root decides - whether the seed or the federated catalog owns it. Anything else - is relative to the seeded node. */ - if(bNoPath){ - strcpyz(sUri, pCat->sSeedUri, uLen); - return DasNode_subNode(pRoot, pCat->sSeedUri, NULL, NULL); - } - if(pCat->bFallThrough && pCat->bPrefixGiven) - snprintf(sUri, uLen, "%s%s", pCat->sPrefix, sPath); - else - DasCat_expand(sPath, sUri, uLen); - if(strpfx(sUri, "tag:")||strpfx(sUri, pCat->sSeedUri)) - return DasNode_subNode(pRoot, sUri, NULL, NULL); - - DasNode* pSeed = DasNode_subNode(pRoot, pCat->sSeedUri, NULL, NULL); - if(pSeed == NULL){ - strcpyz(sUri, pCat->sSeedUri, uLen); - return NULL; - } - const char* sSep = DasNode_isCatalog(pSeed) ? DasNode_pathSep(pSeed) : "/"; - snprintf(sUri, uLen, "%s%s%s", pCat->sSeedUri, sSep ? sSep : "/", sPath); - return DasNode_subNode(pSeed, sPath, NULL, NULL); - } - - if(pCat->sRootUrl[0] != '\0'){ - DasNode* pRoot = DasCat_root(pCat); - if(pRoot == NULL) return NULL; - if(bNoPath){ - strcpyz(sUri, pCat->sRootUrl, uLen); - return pRoot; - } - strcpyz(sUri, sPath, uLen); - return DasNode_subNode(pRoot, sPath, NULL, NULL); - } - - /* The federated catalog under the prefix */ - if(bNoPath){ - /* The prefix with its trailing separator removed names the container itself */ - strcpyz(sUri, pCat->sPrefix, uLen); - size_t u = strlen(sUri); - while((u > 0) && ((sUri[u-1] == '/')||(sUri[u-1] == ':'))) sUri[--u] = '\0'; - } - else if(strpfx(sPath, "tag:")||strpfx(sPath, "site:")||strpfx(sPath, "test:")) - DasCat_expand(sPath, sUri, uLen); - else - snprintf(sUri, uLen, "%s%s", pCat->sPrefix, sPath); - - return DasCat_keep(pCat, new_RootNode(sUri, pCat->pMgr, pCat->sAgent)); -} - -static const char* DasCat_lastUri(const DasCat* pCat){ return pCat->sLastUri; } - -/* A -r ROOT or -g URI=URL value, or their environment stand-ins, turned into - * DasCat policy. URI=URL seeds the catalog: as the only root for -r, as a - * graft for -g. For -r a URL or file alone is a detached root and anything - * else is a URI prefix; -g accepts only URI=URL. The URL side of a seed - * always has a scheme or looks like a file, which is how the '=' that - * matters is found when the URI itself contains one. Returns an error text - * or NULL. */ -static const char* applyRootArg(DasCat* pCat, const char* sVal, bool bGraft) + * DasCat (das3/catalog.h) owns the roots, the credentials, the agent and the + * memory of every node loaded, and resolves paths under one root policy. + * What stays here is the command line spelling of that policy and the + * wording of a failed lookup. */ + + +/* A -r ROOT or -m URI=URL value, or their environment stand-ins, turned into + * catalog policy. URI=URL names a node: the whole catalog for -r, a mount + * on the federated catalog for -m. For -r a URL or file alone is a + * nameless root and anything else is a URI prefix; -m accepts only URI=URL. + * The URL side always has a scheme or looks like a file, which is how the + * '=' that matters is found when the URI itself contains one. Command line + * mistakes are reported here, the catalog logs its own refusals. False on + * either. */ +static bool applyRootArg(DasCat* pCat, const char* sVal, bool bMount) { - const char* sBusy = "only one of -r URL, -r URI=URL or -g URI=URL may be given"; for(const char* pEq = strchr(sVal, '='); pEq != NULL; pEq = strchr(pEq + 1, '=')){ - if(!isDirectUrl(pEq + 1)) continue; - if(DasCat_hasRoot(pCat)) return sBusy; + if(!das_isUrlOrFile(pEq + 1)) continue; if(pEq == sVal){ /* "=URL", a nameless root */ - if(bGraft) return "-g needs URI=URL, a graft without a URI has nowhere to go"; - return DasCat_setRoot(pCat, pEq + 1); + if(bMount){ + fprintf(stderr, PROG ": -m needs URI=URL, a mount without a URI has nowhere to go\n"); + return false; + } + return DasCat_setRoot(pCat, pEq + 1, NULL) == DAS_OKAY; } char sUri[URI_SZ]; size_t uLen = (size_t)(pEq - sVal); if(uLen >= sizeof(sUri)) uLen = sizeof(sUri) - 1; memcpy(sUri, sVal, uLen); sUri[uLen] = '\0'; - return DasCat_seed(pCat, sUri, pEq + 1, bGraft); + if(bMount) return DasCat_mount(pCat, pEq + 1, sUri) == DAS_OKAY; + return DasCat_setRoot(pCat, pEq + 1, sUri) == DAS_OKAY; + } + if(bMount){ + fprintf(stderr, PROG ": -m needs URI=URL, see -h\n"); + return false; } - if(bGraft) return "-g needs URI=URL, see -h"; - if(strchr(sVal, '=') != NULL) return "the part after '=' must be a URL or a file"; - if(isDirectUrl(sVal)){ - if(DasCat_hasRoot(pCat)) return sBusy; - return DasCat_setRoot(pCat, sVal); + if(strchr(sVal, '=') != NULL){ + fprintf(stderr, PROG ": the part after '=' must be a URL or a file\n"); + return false; } - return DasCat_setPrefix(pCat, sVal); + if(das_isUrlOrFile(sVal)) return DasCat_setRoot(pCat, sVal, NULL) == DAS_OKAY; + return DasCat_setPrefix(pCat, sVal) == DAS_OKAY; } /* From a node the user named, get the HttpStreamSrc to query. Containers @@ -1959,46 +1773,42 @@ static DasNode* pickSource(DasNode* pNode, char* sErr, size_t uErr) return pSub; } -/* A path did not resolve. Resolve its parent and say which child was the - * problem and what the parent offers. Call right after the failed - * DasCat_resolve(), while DasCat_lastUri() still describes it. */ +/* A path did not resolve. Say whether the policy or the catalog is to + * blame, and for the catalog, which child was the problem and what its + * parent offers. Call right after the failed DasCat_resolve(). */ static void explainUnresolved(DasCat* pCat, const char* sPath) { char sFullUri[URI_SZ]; strcpyz(sFullUri, DasCat_lastUri(pCat), sizeof(sFullUri)); - /* The common mistake with a restricted root is asking for something it - does not cover, say so instead of hunting for a parent */ - if((pCat->sSeedUrl[0] != '\0') && !pCat->bFallThrough && !strpfx(sFullUri, pCat->sSeedUri)){ - daslog_error_v("'%s' is outside the -r root %s, use -g to patch the " - "federated catalog instead", sFullUri, pCat->sSeedUri); + /* The common mistake with a named root is asking for something it does + not cover, say so instead of hunting for a parent */ + if(!DasCat_covers(pCat, sFullUri)){ + daslog_error_v("'%s' is outside the -r root %s, use -m to mount it on the " + "federated catalog instead", sFullUri, DasCat_rootUri(pCat)); return; } - char sParent[URI_SZ]; - strcpyz(sParent, sPath ? sPath : "", sizeof(sParent)); - size_t u = strlen(sParent); - while((u > 0) && ((sParent[u-1] == '/')||(sParent[u-1] == ':'))) sParent[--u] = '\0'; - char* pLast = strrchr(sParent, '/'); - if((pLast == NULL)||(pLast == sParent)){ + const char* sRest = NULL; + DasNode* pNode = DasCat_nearest(pCat, &sRest); + if((pNode == NULL)||(sRest == NULL)||(sRest[0] == '\0')){ daslog_error_v("Couldn't resolve catalog path '%s'", sFullUri); return; } - *pLast = '\0'; - const char* sChild = pLast + 1; - - DasNode* pNode = DasCat_resolve(pCat, sParent); const char* sParentUri = DasCat_lastUri(pCat); - if((pNode == NULL)||!DasNode_isCatalog(pNode)){ - daslog_error_v("Couldn't resolve catalog path '%s'", sFullUri); - return; - } + + /* Only the first component below the deepest node that loaded can be + the culprit */ + char sChild[NAME_SZ]; + strcpyz(sChild, sRest, sizeof(sChild)); + char* pSlash = strchr(sChild, '/'); + if(pSlash != NULL) *pSlash = '\0'; char sMsg[ERR_SZ]; size_t uKids = DasNode_subCount(pNode); size_t n = 0; bool bFound = false; - for(u = 0; u < uKids; ++u){ + for(size_t u = 0; u < uKids; ++u){ const char* sName = DasNode_subName(pNode, u); if(strieq(sName, sChild)){ const char* sType = jstr(DasNode_subStub(pNode, u), D2FRAG_TYPE); @@ -2015,19 +1825,17 @@ static void explainUnresolved(DasCat* pCat, const char* sPath) if(!bFound) n = snprintf(sMsg, sizeof(sMsg), "No child named '%s' under %s", sChild, sParentUri); if(uKids > 0){ n += snprintf(sMsg + n, sizeof(sMsg) - n, ". Children:"); - for(u = 0; (u < uKids) && (n < sizeof(sMsg)); ++u) + for(size_t u = 0; (u < uKids) && (n < sizeof(sMsg)); ++u) n += snprintf(sMsg + n, sizeof(sMsg) - n, " %s", DasNode_subName(pNode, u)); } daslog_error_v("%s", sMsg); } -/* Everything main() owns, in dependency order: nodes borrow the credential - * manager, so they go first. Returns its argument so exits read as one line. */ +/* Everything main() owns. The catalog frees its nodes, then the credential + * manager they borrow. Returns its argument so exits read as one line. */ static int finish(DasCat* pCat, int nRet) { - DasCat_free(pCat); - if(pCat->pMgr != NULL) del_CredMngr(pCat->pMgr); - pCat->pMgr = NULL; + del_DasCat(pCat); return nRet; } @@ -2041,7 +1849,7 @@ static int streamUrl(DasCat* pCat, const char* sUrl, float rTimeout) daslog_info_v("Requesting %s", sUrl); - if(!das_http_getBody(sUrl, pCat->sAgent, pCat->pMgr, &res, rTimeout)){ + if(!das_http_getBody(sUrl, DasCat_agent(pCat), DasCat_credMngr(pCat), &res, rTimeout)){ int nRet = RET_SERVER; if((res.nCode == 401)||(res.nCode == 403)) nRet = RET_AUTH; else if(res.nCode == 400) nRet = RET_QUERY; @@ -2138,14 +1946,13 @@ int main(int argc, char** argv) char sAgent[64]; snprintf(sAgent, sizeof(sAgent), PROG " (das2C %s)", das_version()); - DasCat cat; - DasCat_init(&cat, NULL, sAgent); /* credentials attach once the manager exists */ + DasCat* pCat = NULL; /* made once the arguments are in, see below */ const char* sCmd = NULL; const char* sPath = NULL; const char* sCacheArg = NULL; const char* sRootArg = NULL; - const char* sGraftArg = NULL; + const char* sMountArg = NULL; bool bHelp = false; bool bFresh = false; bool bQuiet = false; @@ -2168,8 +1975,8 @@ int main(int argc, char** argv) if(strcmp(sArg, "-v") == 0){ daslog_setlevel(DASLOG_INFO); g_bVerbose = true; g_uShow = SHOW_ALL; continue; } if(strcmp(sArg, "-f") == 0){ bFresh = true; continue; } if(strcmp(sArg, "--root") == 0) sArg = "-r"; - if(strcmp(sArg, "--graft") == 0) sArg = "-g"; - if((strcmp(sArg, "-r") == 0)||(strcmp(sArg, "-g") == 0)||(strcmp(sArg, "-s") == 0)|| + if(strcmp(sArg, "--mount") == 0) sArg = "-m"; + if((strcmp(sArg, "-r") == 0)||(strcmp(sArg, "-m") == 0)||(strcmp(sArg, "-s") == 0)|| (strcmp(sArg, "-C") == 0)||(strcmp(sArg, "-p") == 0)||(strcmp(sArg, "-t") == 0)|| (strcmp(sArg, "-l") == 0)){ if(i + 1 >= argc){ @@ -2179,7 +1986,7 @@ int main(int argc, char** argv) const char* sVal = argv[++i]; switch(sArg[1]){ case 'r': sRootArg = sVal; break; - case 'g': sGraftArg = sVal; break; + case 'm': sMountArg = sVal; break; case 's': if(!parseShow(sVal, &g_uShow)){ fprintf(stderr, PROG ": -s takes a list from coords, data, formats, options, " @@ -2225,33 +2032,47 @@ int main(int argc, char** argv) if(bHelp){ prnHelp(sCmd); return RET_OK; } g_nCols = termWidth(); - /* Environment fallbacks for -r and -g, for people who live in one test - catalog. The root is applied first so a graft can complain about a + /* Credentials: saved ones are tried before the terminal prompt. The + catalog owns the manager and hands it to every node and request. */ + char sCredFile[256] = {'\0'}; + const char* sHome = das_userhome(); + if(sHome && sHome[0]) + snprintf(sCredFile, sizeof(sCredFile), "%s" DAS_DSEPS DAS_DIR DAS_DSEPS CRED_FILE, sHome); + DasCredMngr* pMgr = new_CredMngr(sCredFile[0] ? sCredFile : NULL); + if(sCredFile[0]){ + /* CredMngr_load logs an error for a missing file even though its + contract says a missing file is not an error. Most users have no + saved credentials, so look first. */ + FILE* pTest = fopen(sCredFile, "r"); + if(pTest != NULL){ + fclose(pTest); + CredMngr_load(pMgr, NULL, NULL); + } + } + pCat = new_DasCat(pMgr, sAgent); + + /* Environment fallbacks for -r and -m, for people who live in one test + catalog. The root is applied first so a mount can complain about a conflicting one. */ const char* sEnv = getenv(ROOT_ENV); if((sRootArg == NULL) && (sEnv != NULL) && (sEnv[0] != '\0')) sRootArg = sEnv; - sEnv = getenv(GRAFT_ENV); - if((sGraftArg == NULL) && (sEnv != NULL) && (sEnv[0] != '\0')) sGraftArg = sEnv; - const char* sBad = NULL; - if((sRootArg != NULL) && ((sBad = applyRootArg(&cat, sRootArg, false)) != NULL)){ - fprintf(stderr, PROG ": %s\n", sBad); - return RET_USAGE; - } - if((sGraftArg != NULL) && ((sBad = applyRootArg(&cat, sGraftArg, true)) != NULL)){ - fprintf(stderr, PROG ": %s\n", sBad); - return RET_USAGE; - } + sEnv = getenv(MOUNT_ENV); + if((sMountArg == NULL) && (sEnv != NULL) && (sEnv[0] != '\0')) sMountArg = sEnv; + if((sRootArg != NULL) && !applyRootArg(pCat, sRootArg, false)) + return finish(pCat, RET_USAGE); + if((sMountArg != NULL) && !applyRootArg(pCat, sMountArg, true)) + return finish(pCat, RET_USAGE); if(sCmd == NULL){ fprintf(stderr, PROG ": no command given, use -h for help\n"); - return RET_USAGE; + return finish(pCat, RET_USAGE); } if(strcmp(sCmd, "roots") == 0){ size_t uRoots = 0; const char** psRoots = das_root_urls(&uRoots); for(size_t u = 0; u < uRoots; ++u) printf("%s\n", psRoots[u]); - return RET_OK; + return finish(pCat, RET_OK); } bool bNode = (strcmp(sCmd, "node") == 0)||(strcmp(sCmd, "ls") == 0); @@ -2260,11 +2081,11 @@ int main(int argc, char** argv) bool bData = (strcmp(sCmd, "data") == 0)||(strcmp(sCmd, "get") == 0); if(!bNode && !bJson && !bUrl && !bData){ fprintf(stderr, PROG ": unknown command '%s', use -h for help\n", sCmd); - return RET_USAGE; + return finish(pCat, RET_USAGE); } - if((bJson || bUrl || bData) && (sPath == NULL) && !DasCat_hasRoot(&cat)){ + if((bJson || bUrl || bData) && (sPath == NULL) && !DasCat_hasRoot(pCat) && (DasCat_mountCount(pCat) == 0)){ fprintf(stderr, PROG ": the %s command needs a PATH, use -h for help\n", sCmd); - return RET_USAGE; + return finish(pCat, RET_USAGE); } if(bNode && (uKeyVals > 0)) daslog_warn("KEY=VALUE arguments are ignored by the node command"); @@ -2274,24 +2095,7 @@ int main(int argc, char** argv) char sCacheDir[512] = {'\0'}; if(sCacheArg == NULL) defCacheDir(sCacheDir, sizeof(sCacheDir)); else if(!strieq(sCacheArg, "none")) strcpyz(sCacheDir, sCacheArg, sizeof(sCacheDir)); - if(sCacheDir[0] != '\0') DasNode_setCache(sCacheDir, bFresh ? 0 : DEF_CACHE_AGE); - - /* Credentials: saved ones are tried before the terminal prompt */ - char sCredFile[256] = {'\0'}; - const char* sHome = das_userhome(); - if(sHome && sHome[0]) - snprintf(sCredFile, sizeof(sCredFile), "%s" DAS_DSEPS DAS_DIR DAS_DSEPS CRED_FILE, sHome); - cat.pMgr = new_CredMngr(sCredFile[0] ? sCredFile : NULL); - if(sCredFile[0]){ - /* CredMngr_load logs an error for a missing file even though its - contract says a missing file is not an error. Most users have no - saved credentials, so look first. */ - FILE* pTest = fopen(sCredFile, "r"); - if(pTest != NULL){ - fclose(pTest); - CredMngr_load(cat.pMgr, NULL, NULL); - } - } + DasCat_setCache(pCat, sCacheDir, bFresh ? 0 : DEF_CACHE_AGE); /* empty dir: off */ /* A fragment on the path is only meaningful for the json command */ char sPathBuf[1024] = {'\0'}; @@ -2301,21 +2105,22 @@ int main(int argc, char** argv) char* pHash = strchr(sPathBuf, '#'); if(pHash != NULL){ *pHash = '\0'; sFrag = pHash + 1; } /* "a/b/" means the same as "a/b", shell completion adds the slash */ - if(!isDirectUrl(sPathBuf)){ + if(!das_isUrlOrFile(sPathBuf)){ size_t u = strlen(sPathBuf); while((u > 1) && (sPathBuf[u-1] == '/') && (sPathBuf[u-2] != ':')) sPathBuf[--u] = '\0'; } sPath = sPathBuf; } - DasNode* pNode = DasCat_resolve(&cat, sPath); + DasNode* pNode = DasCat_resolve(pCat, sPath); if(pNode == NULL){ - explainUnresolved(&cat, sPath); - return finish(&cat, RET_NODE); + explainUnresolved(pCat, sPath); + return finish(pCat, RET_NODE); } - const char* sFullUri = DasCat_lastUri(&cat); + const char* sFullUri = DasCat_lastUri(pCat); char sShow[1200]; - bool bNoUri = (cat.sRootUrl[0] != '\0') || isDirectUrl(sPath); /* URL or file, not a catalog URI */ + /* Under a nameless root or a path by location there is no catalog URI to show */ + bool bNoUri = ((DasCat_rootUrl(pCat) != NULL) && (DasCat_rootUri(pCat) == NULL)) || das_isUrlOrFile(sPath); const char* sLabel = nodeUri(pNode, bNoUri, sFullUri, sShow, sizeof(sShow)); int nRet = RET_OK; @@ -2330,7 +2135,7 @@ int main(int argc, char** argv) char* sPretty = (char*)DasJdo_writePretty(pJdo, " ", "\n", NULL); if(sPretty){ fputs(sPretty, stdout); fputs("\n", stdout); free(sPretty); } } - return finish(&cat, nRet); + return finish(pCat, nRet); } if(bNode){ @@ -2398,20 +2203,20 @@ int main(int argc, char** argv) if(g_bVerbose) printf("From: %s\n", DasNode_srcUrl(pNode)); printf("\n" PROG " has no detailed printer for this node type, try the json command.\n"); } - return finish(&cat, nRet); + return finish(pCat, nRet); } /* url or data */ DasNode* pSrc = pickSource(pNode, sErr, sizeof(sErr)); if(pSrc == NULL){ daslog_error_v("%s", sErr); - return finish(&cat, RET_NODE); + return finish(pCat, RET_NODE); } DasHttpSrc src; if(!DasHttpSrc_init(&src, pSrc, sErr, sizeof(sErr))){ daslog_error_v("%s", sErr); - return finish(&cat, RET_NODE); + return finish(pCat, RET_NODE); } /* Order matters: example first, then explicit keys override it, and -p @@ -2437,7 +2242,7 @@ int main(int argc, char** argv) if(bSec){ if(!isDigits(pDot + 1)){ daslog_error_v("Examples are numbered, there is no '%s'", sArg); - return finish(&cat, RET_QUERY); + return finish(pCat, RET_QUERY); } sNum = pDot + 1; } @@ -2459,13 +2264,13 @@ int main(int argc, char** argv) if(nExample >= 0){ if(!DasHttpSrc_applyExample(&src, (size_t)nExample, aExRaw, &uExRaw, MAX_RAW, sErr, sizeof(sErr))){ daslog_error_v("%s", sErr); - return finish(&cat, RET_QUERY); + return finish(pCat, RET_QUERY); } } for(size_t u = 0; u < uExRaw; ++u){ if(!DasHttpSrc_outSet(&src, aExRaw[u].sName, aExRaw[u].sVal, sErr, sizeof(sErr))){ daslog_error_v("%s", sErr); - return finish(&cat, RET_QUERY); + return finish(pCat, RET_QUERY); } } for(size_t u = 0; u < uKeyVals; ++u){ @@ -2484,23 +2289,23 @@ int main(int argc, char** argv) } if(!DasHttpSrc_set(&src, sKey, sVal, sErr, sizeof(sErr))){ daslog_error_v("%s", sErr); - return finish(&cat, RET_QUERY); + return finish(pCat, RET_QUERY); } } if(!DasHttpSrc_build(&src, sErr, sizeof(sErr))){ daslog_error_v("%s", sErr); - return finish(&cat, RET_QUERY); + return finish(pCat, RET_QUERY); } for(size_t u = 0; u < uRaw; ++u){ if(!DasHttpSrc_outSet(&src, aRaw[u].sName, aRaw[u].sVal, sErr, sizeof(sErr))){ daslog_error_v("%s", sErr); - return finish(&cat, RET_QUERY); + return finish(pCat, RET_QUERY); } } if(!DasHttpSrc_checkRequired(&src, sErr, sizeof(sErr))){ daslog_error_v("%s", sErr); - return finish(&cat, RET_QUERY); + return finish(pCat, RET_QUERY); } char sUrl[URL_SZ]; @@ -2512,7 +2317,7 @@ int main(int argc, char** argv) else{ printf("%s\n", sUrl); } - return finish(&cat, nRet); + return finish(pCat, nRet); } /* data: try each base URL in turn until one answers. A server that @@ -2525,12 +2330,12 @@ int main(int argc, char** argv) nRet = RET_QUERY; break; } - nRet = streamUrl(&cat, sUrl, rTimeout); + nRet = streamUrl(pCat, sUrl, rTimeout); if((nRet != RET_SERVER)||(u + 1 >= uBases)) break; daslog_warn_v("Trying alternate server %s", DasHttpSrc_baseUrl(&src, u + 1)); } - finish(&cat, nRet); + finish(pCat, nRet); das_finish(); return nRet; } From f012858082487cfb7b941bf162a8473e887b7a2c Mon Sep 17 00:00:00 2001 From: cpiker Date: Sun, 25 Oct 2026 20:26:49 -0500 Subject: [PATCH 04/18] http: bug fixes and buffer expansion URL parts that do not fit are refused, never truncated, Path and query limits raised to 255 and 3071. Developed-with: Claude Fable 5.1 --- buildfiles/Darwin.mak | 2 + buildfiles/Linux.mak | 2 + buildfiles/Windows.mak | 2 + buildfiles/cmake/das2CTests.cmake | 1 + buildfiles/sources.mak | 2 +- das3/http.c | 113 +++++++++---------- das3/http.h | 37 ++++--- das3/io.h | 2 +- test/TestHttp.c | 175 ++++++++++++++++++++++++++++++ 9 files changed, 257 insertions(+), 79 deletions(-) create mode 100644 test/TestHttp.c diff --git a/buildfiles/Darwin.mak b/buildfiles/Darwin.mak index bb2787b..85dd4fc 100644 --- a/buildfiles/Darwin.mak +++ b/buildfiles/Darwin.mak @@ -376,6 +376,8 @@ test_main: $(BD) $(BD)/$(TARG).a $(BUILD_TEST_PROGS) $(BULID_UTIL_PROGS) @$(BD)/TestCredMngr $(BD) @echo "INFO: Running unit test for catalog sessions over local nodes, $(BD)/TestCat..." @$(BD)/TestCat + @echo "INFO: Running unit test for the URL parser, $(BD)/TestHttp..." + @$(BD)/TestHttp @echo "INFO: Running unit test for exception names and send.h writers, $(BD)/TestOob..." @$(BD)/TestOob $(BD)/TestOob send 2 > $(BD)/oob_send.d2s && $(BD)/TestOob read $(BD)/oob_send.d2s diff --git a/buildfiles/Linux.mak b/buildfiles/Linux.mak index 951ec89..80e857a 100644 --- a/buildfiles/Linux.mak +++ b/buildfiles/Linux.mak @@ -374,6 +374,8 @@ test_main: $(BD) $(BD)/$(TARG).a $(BUILD_TEST_PROGS) $(BUILD_UTIL_PROGS) @$(BD)/TestCredMngr $(BD) @echo "INFO: Running unit test for catalog sessions over local nodes, $(BD)/TestCat..." @$(BD)/TestCat + @echo "INFO: Running unit test for the URL parser, $(BD)/TestHttp..." + @$(BD)/TestHttp @echo "INFO: Running unit test for exception names and send.h writers, $(BD)/TestOob..." @$(BD)/TestOob $(BD)/TestOob send 2 > $(BD)/oob_send.d2s && $(BD)/TestOob read $(BD)/oob_send.d2s diff --git a/buildfiles/Windows.mak b/buildfiles/Windows.mak index ce6b65f..eb7d98f 100644 --- a/buildfiles/Windows.mak +++ b/buildfiles/Windows.mak @@ -390,6 +390,8 @@ test_main: @$(BD)\TestCredMngr.exe $(BD) @echo INFO: Running unit test for catalog sessions over local nodes, $(BD)\TestCat... @$(BD)\TestCat.exe + @echo INFO: Running unit test for the URL parser, $(BD)\TestHttp... + @$(BD)\TestHttp.exe @echo INFO: Running unit test for exception names and send.h writers, $(BD)\TestOob... @$(BD)\TestOob.exe $(BD)\TestOob.exe send 2 > $(BD)\oob_send.d2s diff --git a/buildfiles/cmake/das2CTests.cmake b/buildfiles/cmake/das2CTests.cmake index 969fc4e..3a78692 100644 --- a/buildfiles/cmake/das2CTests.cmake +++ b/buildfiles/cmake/das2CTests.cmake @@ -65,6 +65,7 @@ _das_test(TestFilter main TestFilter) _das_test(TestUri main TestUri "${_bd}/uri_tplt") _das_test(TestCredMngr main TestCredMngr "${_bd}") _das_test(TestCat main TestCat) +_das_test(TestHttp main TestHttp) _das_test(TestOob main TestOob) foreach(_v 2 3) if(_v EQUAL 2) diff --git a/buildfiles/sources.mak b/buildfiles/sources.mak index ebc1525..6f793aa 100644 --- a/buildfiles/sources.mak +++ b/buildfiles/sources.mak @@ -61,7 +61,7 @@ CDF_UTIL_SRCS = utilities/das3_cdf.c utilities/das3_from_cdf.c \ utilities/das2_from_cdf.c TEST_SRCS = test/TestUnits.c test/TestArray.c test/TestDs.c test/TestBuilder.c \ - test/TestAuth.c test/TestCatalog.c test/TestCat.c test/TestTT2000.c test/ex_das_cli.c \ + test/TestAuth.c test/TestCatalog.c test/TestCat.c test/TestHttp.c test/TestTT2000.c test/ex_das_cli.c \ test/ex_das_ephem.c test/TestCredMngr.c test/TestV3Read.c test/TestFeed.c \ test/TestProp.c \ test/TestIter.c test/TestUri.c test/TestFilter.c test/TestValue.c \ diff --git a/das3/http.c b/das3/http.c index bb80ab3..bb29d92 100644 --- a/das3/http.c +++ b/das3/http.c @@ -91,8 +91,6 @@ char html5[256] = { #define HTTP_NotFound 404 #define HTTP_Error 500 -#define _QDEF(x) #x -#define QDEF(x) _QDEF(x) /* ************************************************************************* */ /* The global SSL context and mutexes for manipulating it. We use lazy @@ -272,6 +270,29 @@ char* das_ssl_getErr(const void* vpSsl, int nRet) /* ************************************************************************* */ /* Parsing a URL */ +/* Copy input into a fixed URL field up to the first stop character or the + * end of input. A part that does not fit is an error, never a truncation: + * a shortened path or query would reach the server as a different request, + * and a shortened dataset would match the wrong saved password. Returns + * where the copy stopped, or NULL with pRes->sError set. */ +static const char* _das_url_field( + DasHttpResp* pRes, char* sOut, size_t uMax, const char* pIn, + const char* sStop, const char* sWhat, const char* sUrl +){ + size_t u = 0; + while((*pIn != '\0') && (strchr(sStop, *pIn) == NULL)){ + if(u == uMax){ + pRes->sError = das_string( + "The %s in URL %s is longer than %zu bytes", sWhat, sUrl, uMax + ); + return NULL; + } + sOut[u++] = *pIn++; + } + sOut[u] = '\0'; + return pIn; +} + bool DasHttpResp_init(DasHttpResp* pRes, const char* sUrl) { struct das_url* pUrl = &(pRes->url); @@ -279,82 +300,53 @@ bool DasHttpResp_init(DasHttpResp* pRes, const char* sUrl) pRes->nSockFd = -1; pRes->nCode = -1; - pRes->url.sPort[0] = '8'; pRes->url.sPort[1] = '0'; - /* Get the scheme, this is a PITA but I don't want a large library - * dependency and uriparser doesn't want to deal with utf-8 natively. - * I'm sure curl would take care of it nicely, but does that exist on - * windows? */ - char* pOut = pUrl->sScheme; - const char* pIn = sUrl; - while((*pIn != '\0')&&(*pIn != ':')&&((pOut - pUrl->sScheme) <= DASURL_SZ_SCHEME )){ - *pOut = *pIn; ++pOut; ++pIn; - } - if((strcmp("http", pUrl->sScheme)!=0 )&&(strcmp("https", pUrl->sScheme)!=0)){ + /* Hand rolled on purpose: uriparser does not take UTF-8 natively and + * curl is a large dependency to carry for a handful of fields. */ + const char* pIn = _das_url_field(pRes, pUrl->sScheme, DASURL_SZ_SCHEME, sUrl, ":", "scheme", sUrl); + if(pIn == NULL) return false; + if((strcmp("http", pUrl->sScheme) != 0)&&(strcmp("https", pUrl->sScheme) != 0)){ pRes->sError = das_string("Unknown scheme, %s", pUrl->sScheme); return false; } - if(strcmp("https", pUrl->sScheme) == 0){ - pUrl->sPort[0] = '4'; pUrl->sPort[1] = '4'; pUrl->sPort[2] = '3'; - } - else{ - pUrl->sPort[0] = '8'; pUrl->sPort[1] = '0'; - } + strcpy(pUrl->sPort, (strcmp("https", pUrl->sScheme) == 0) ? "443" : "80"); while(*pIn == ':') pIn++; while(*pIn == '/') pIn++; /* Advance past a couple /'s or four */ - pOut = pUrl->sHost; - while( (*pIn != '\0')&&(*pIn != ':')&&(*pIn != '/')&&(*pIn != '?')&& - ((pOut - pUrl->sHost) <= DASURL_SZ_HOST)){ - *pOut = *pIn; ++pOut; ++pIn; - } - if((strlen(pUrl->sHost) < 1)||((pOut - pUrl->sHost) == (DASURL_SZ_HOST))){ + pIn = _das_url_field(pRes, pUrl->sHost, DASURL_SZ_HOST, pIn, ":/?", "host", sUrl); + if(pIn == NULL) return false; + if(pUrl->sHost[0] == '\0'){ pRes->sError = das_string("Invalid host in URL %s", sUrl); return false; } - char sPort[32] = {'\0'}; - pOut = sPort; - int nPort; if(*pIn == ':'){ - ++pIn; /* The colon is a delimiter, not part of the port number. Copying - it made atoi() return 0 and every URL with an explicit port - was rejected as invalid. */ - while( (*pIn != '\0')&&(*pIn != '/')&&(*pIn != '?')&&((pOut - sPort) < 31) ){ - *pOut = *pIn; ++pOut; ++pIn; - } - nPort = 0; - if((strlen(sPort) < 1) || ((nPort = atoi(sPort)) == 0) || - (nPort > 65535) ){ + /* The colon is a delimiter, not part of the port number */ + char sPort[DASURL_SZ_PORT + 1]; + pIn = _das_url_field(pRes, sPort, DASURL_SZ_PORT, pIn + 1, "/?", "port", sUrl); + if(pIn == NULL) return false; + int nPort = atoi(sPort); + if((nPort < 1)||(nPort > 65535)){ pRes->sError = das_string("Invalid port in URL %s", sUrl); return false; } - strncpy(pUrl->sPort, sPort, 7); + strcpy(pUrl->sPort, sPort); } - /* don't skip the / it's part of the path */ - - /* Skipping fragments for now */ - pOut = pUrl->sPath; /* Note this can be zero */ - while( (*pIn != '\0')&&(*pIn != '?')&&((pOut - pUrl->sPath)sPath, DASURL_SZ_PATH, pIn, "?", "path", sUrl); + if(pIn == NULL) return false; if(*pIn == '?') ++pIn; - pOut = pUrl->sQuery; - while((*pIn != '\0')&&((pOut - pUrl->sQuery)sQuery, "dataset=")) != NULL){ + pIn = _das_url_field(pRes, pUrl->sDataset, DASURL_SZ_DATASET, pIn + 8, "&", "dataset", sUrl); + if(pIn == NULL) return false; } return true; } @@ -1037,11 +1029,6 @@ bool das_http_getBody( DasHttpResp_clear(pRes); /* Sets nSockFd to -1 */ - if(strlen(sUrl) > DASURL_SZ_QUERY){ - pRes->sError = das_string("URL is greater than " QDEF(DASURL_SZ_QUERY) " bytes"); - return false; - } - struct timeval tv = {0, 0}; if(rConSec > 0.0){ tv.tv_sec = (long)rConSec; diff --git a/das3/http.h b/das3/http.h index 57bb251..44d581d 100644 --- a/das3/http.h +++ b/das3/http.h @@ -34,10 +34,13 @@ extern "C" { * @{ */ +/* Sizes of the parts of struct das_url, not counting the terminator. A URL + * with a part longer than this is refused by DasHttpResp_init(), it is never + * sent truncated. */ #define DASURL_SZ_SCHEME 31 #define DASURL_SZ_HOST 63 -#define DASURL_SZ_PATH 127 -#define DASURL_SZ_QUERY 1023 +#define DASURL_SZ_PATH 255 +#define DASURL_SZ_QUERY 3071 #define DASURL_SZ_DATASET 63 #define DASURL_SZ_PORT 7 @@ -124,10 +127,9 @@ DAS_API char* das_url_toStr(const struct das_url* pUrl, char* sBuf, size_t uLen) /** Initialize all fields in an http response to default values. * - * All char fields are set to zero, the file descriptor (nSockFd) is set to - * -1 and the response code is set to zero. This function is called - * automatically by http_getBodySocket() so there is usually no need to use it - * directly. + * All char fields are set to zero, the file descriptor (nSockFd) and the + * response code are set to -1. This function is called automatically by + * das_http_getBody() so there is usually no need to use it directly. * * @param pRes The response to clear * @memberof DasHttpResp @@ -145,9 +147,13 @@ DAS_API void DasHttpResp_freeFields(DasHttpResp* pRes); * * The URL will be parsed into a das_url structure and stored internally. * - * @param pRes The response to initialized + * @param pRes The response to initialize * @param sUrl The fully qualified URL including the scheme, host, port (if any), * path and query parameters. + * @returns true if every part fit its field in struct das_url. False, with + * pRes->sError set, if the scheme is not http or https, the host + * or port is bad, or a part is longer than its DASURL_SZ_ limit. + * Nothing is truncated. * @memberof DasHttpResp */ DAS_API bool DasHttpResp_init(DasHttpResp* pRes, const char* sUrl); @@ -182,14 +188,17 @@ DAS_API bool DasHttpResp_useSsl(DasHttpResp* pRes); * socket descriptor or the SSL connection when it is finished reading the * message body. * - * Das2 Note: Since das2 servers can request different authentication for - * each dataset, the get string is inspected for the 'server=dataset' pair. If - * found the URL saved in the credentials manager will be - * http://SERVER/path?dataset=DATASET instead of just http://SERVER/path. - * This a bit of a hack and a better solution should be found in the future. + * das2.2 Note: The das2.2 protocol lets a server require a different + * login for each dataset it serves (das2.2 ICD, https://zenodo.org/records/3588535). + * So when the query string carries a dataset= value, credentials are looked + * up and saved under http://SERVER/path?dataset=DATASET rather than + * http://SERVER/path. This is part of the das2.2 interface, which das2C + * supports alongside das3. * - * @param sUrl The location to connect to, will be URL encoded before - * transmission + * @param sUrl The location to connect to, sent as given. No encoding is + * applied here, so reserved characters in parameter values must + * already be percent encoded by the caller. Each part must fit + * its field in struct das_url, see DasHttpResp_init(). * * @param sAgent The user agent string you wish to send to the server. If NULL * then the string "libdas/3" is sent. diff --git a/das3/io.h b/das3/io.h index d676493..2426d1d 100644 --- a/das3/io.h +++ b/das3/io.h @@ -236,7 +236,7 @@ DAS_API DasIO* new_DasIO_str(const char* sProg, char* sbuf, size_t len, const ch * * You should also setup the SSL connection with the SSL_MODE_AUTO_RETRY option * to SSL_set_mode or SSL_CTX_set_mode to let the openssl library handle - * session renegotiation transparently. The http_getBodySocket() call does + * session renegotiation transparently. The das_http_getBody() call does * initialize any SSL structures it generates in auto-retry mode. * * @param sProg A spot to store the name of the program creating the file diff --git a/test/TestHttp.c b/test/TestHttp.c new file mode 100644 index 0000000..433a85d --- /dev/null +++ b/test/TestHttp.c @@ -0,0 +1,175 @@ +/** @file TestHttp.c Unit tests for http.h that need no network: today the + * URL parser behind DasHttpResp_init(). */ + +/* Author: Chris Piker , via Claude Fable 5.1 + * + * This file is intended to demonstrate an interface. This is free + * and unencumbered software released into the public domain + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this file, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this file dedicate any and all copyright interest in this file to + * the public domain. We make this dedication for the benefit of the + * public at large and to the detriment of our heirs and successors. We + * intend this dedication to be an overt act of relinquishment in + * perpetuity of all present and future rights to this file under + * copyright law. + * + * THIS FILE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * + * For more information, please refer to + */ + +/* The parser has fixed size fields. The property under test is that a part + * which fits is kept whole and a part which does not is refused with an + * error message, never silently shortened. A shortened query would reach + * the server as a different request and a shortened dataset would match + * the wrong saved password. */ + +#include +#include +#include +#include + +#include +#include + +static int g_nFail = 0; +static int g_nPass = 0; + +#define CHECK(cond, ...) do{ \ + if(cond){ ++g_nPass; } \ + else{ ++g_nFail; printf(" FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } \ +}while(0) + +static bool streq(const char* a, const char* b) +{ + if((a == NULL)||(b == NULL)) return (a == b); + return strcmp(a, b) == 0; +} + +/* Parse into a cleared response. Any error string is freed before the + * next call so a leak checker stays quiet. */ +static bool parse(DasHttpResp* pRes, const char* sUrl) +{ + if(pRes->sError){ free(pRes->sError); pRes->sError = NULL; } + memset(pRes, 0, sizeof(DasHttpResp)); + return DasHttpResp_init(pRes, sUrl); +} + +/* A URL whose named part is exactly uLen characters long */ +static void mkUrl(char* sBuf, size_t uBuf, const char* sPart, size_t uLen) +{ + size_t u = 0; + if(strcmp(sPart, "scheme") == 0){ + for(; u < uLen; ++u) sBuf[u] = 'h'; + snprintf(sBuf + u, uBuf - u, "://host/p"); + memcpy(sBuf, "http", 4); /* still not a valid scheme, length is the point */ + return; + } + if(strcmp(sPart, "host") == 0){ + u = snprintf(sBuf, uBuf, "http://"); + for(size_t i = 0; i < uLen; ++i) sBuf[u++] = 'h'; + snprintf(sBuf + u, uBuf - u, "/p"); + return; + } + if(strcmp(sPart, "path") == 0){ + u = snprintf(sBuf, uBuf, "http://host/"); + for(size_t i = 1; i < uLen; ++i) sBuf[u++] = 'p'; + snprintf(sBuf + u, uBuf - u, "?q=1"); + return; + } + if(strcmp(sPart, "query") == 0){ + u = snprintf(sBuf, uBuf, "http://host/p?"); + for(size_t i = 0; i < uLen; ++i) sBuf[u++] = 'q'; + sBuf[u] = '\0'; + return; + } + if(strcmp(sPart, "dataset") == 0){ + u = snprintf(sBuf, uBuf, "http://host/p?server=dataset&dataset="); + for(size_t i = 0; i < uLen; ++i) sBuf[u++] = 'd'; + snprintf(sBuf + u, uBuf - u, "&start_time=2026"); + return; + } +} + +int main(int argc, char** argv) +{ + das_init(argv[0], DASERR_DIS_EXIT, 0, DASLOG_WARN, NULL); + + DasHttpResp res; + memset(&res, 0, sizeof(DasHttpResp)); + char sUrl[DASURL_SZ_QUERY + 256]; + char sOut[DASURL_SZ_QUERY + 256]; + + printf("INFO: Parsing well formed URLs\n"); + + CHECK(parse(&res, "http://host/path?a=1"), "plain http parses"); + CHECK(streq(res.url.sScheme, "http"), "scheme"); + CHECK(streq(res.url.sHost, "host"), "host"); + CHECK(streq(res.url.sPort, "80"), "default http port, got %s", res.url.sPort); + CHECK(streq(res.url.sPath, "/path"), "path, got %s", res.url.sPath); + CHECK(streq(res.url.sQuery, "a=1"), "query, got %s", res.url.sQuery); + CHECK(res.url.sDataset[0] == '\0', "no dataset"); + CHECK(streq(das_url_toStr(&res.url, sOut, sizeof(sOut)), "http://host/path?a=1"), + "round trip drops the default port, got %s", sOut); + + CHECK(parse(&res, "https://space.physics.uiowa.edu:8443/das/stream/source/tracers/flex" + "?read.time.min=2026-01-01T00%3A00"), "https with port parses"); + CHECK(streq(res.url.sPort, "8443"), "explicit port, got %s", res.url.sPort); + CHECK(streq(res.url.sPath, "/das/stream/source/tracers/flex"), "path, got %s", res.url.sPath); + CHECK(streq(res.url.sQuery, "read.time.min=2026-01-01T00%3A00"), "query, got %s", res.url.sQuery); + CHECK(streq(das_url_toStr(&res.url, sOut, sizeof(sOut)), + "https://space.physics.uiowa.edu:8443/das/stream/source/tracers/flex?read.time.min=2026-01-01T00%3A00"), + "round trip keeps an explicit port, got %s", sOut); + + CHECK(parse(&res, "https://host"), "bare host parses"); + CHECK(streq(res.url.sPort, "443"), "default https port, got %s", res.url.sPort); + CHECK(res.url.sPath[0] == '\0', "empty path"); + CHECK(res.url.sQuery[0] == '\0', "empty query"); + + CHECK(parse(&res, "http://h/x?server=dataset&dataset=Juno/WAV/Survey&start_time=1"), + "das2.2 style query parses"); + CHECK(streq(res.url.sDataset, "Juno/WAV/Survey"), "dataset stops at &, got %s", res.url.sDataset); + + printf("INFO: Refusing malformed URLs\n"); + + CHECK(!parse(&res, "ftp://host/x") && res.sError, "ftp scheme refused"); + CHECK(!parse(&res, "http://") && res.sError, "empty host refused"); + CHECK(!parse(&res, "http://host:99999/x") && res.sError, "port out of range refused"); + CHECK(!parse(&res, "http://host:abc/x") && res.sError, "non numeric port refused"); + CHECK(!parse(&res, "http://host:/x") && res.sError, "empty port refused"); + + printf("INFO: Parts at the size limit fit, one byte over is refused\n"); + + struct { const char* sPart; size_t uMax; } aLim[] = { + {"host", DASURL_SZ_HOST}, {"path", DASURL_SZ_PATH}, + {"query", DASURL_SZ_QUERY}, {"dataset", DASURL_SZ_DATASET} + }; + for(size_t i = 0; i < sizeof(aLim)/sizeof(aLim[0]); ++i){ + mkUrl(sUrl, sizeof(sUrl), aLim[i].sPart, aLim[i].uMax); + CHECK(parse(&res, sUrl), "%s of %zu bytes parses: %s", aLim[i].sPart, aLim[i].uMax, + res.sError ? res.sError : ""); + + mkUrl(sUrl, sizeof(sUrl), aLim[i].sPart, aLim[i].uMax + 1); + bool bOk = parse(&res, sUrl); + CHECK(!bOk, "%s of %zu bytes is refused", aLim[i].sPart, aLim[i].uMax + 1); + CHECK(!bOk && res.sError && strstr(res.sError, aLim[i].sPart), + "refusal names the %s, got: %s", aLim[i].sPart, res.sError ? res.sError : "(null)"); + } + + /* The scheme is checked by value before anything else, so an over long + one is refused either way; make sure it is refused and not overrun */ + mkUrl(sUrl, sizeof(sUrl), "scheme", DASURL_SZ_SCHEME + 1); + CHECK(!parse(&res, sUrl) && res.sError, "over long scheme refused"); + if(res.sError){ free(res.sError); res.sError = NULL; } + + printf("INFO: %d checks passed, %d failed\n", g_nPass, g_nFail); + return (g_nFail == 0) ? 0 : 13; +} From 79e51c190da1ec153dea5b8ad0a2ad5c3935bc0a Mon Sep 17 00:00:00 2001 From: cpiker Date: Sun, 25 Oct 2026 22:30:47 -0500 Subject: [PATCH 05/18] util: common das user dir, cat node caching Everything das2C keeps for a user lives under ~/.das on every platform, or $DAS_USERDIR, or whatever a program gives das_setUserDir(). A new DasCat caches catalog nodes there for an hour unless told otherwise. das_fetch, das3_cdf and das3_info updated to use the same area. Developed-with: Claude Fable 5.1 --- das3/catalog.c | 25 +++++++++++++++++- das3/catalog.h | 37 ++++++++++++++++++++------ das3/node.c | 23 ++++------------- das3/node.h | 19 -------------- das3/node_priv.h | 13 +++++----- das3/util.c | 25 ++++++++++++++++++ das3/util.h | 27 +++++++++++++++++++ test/TestCat.c | 60 +++++++++++++++++++++++++++++++++++++++++++ utilities/das3_cdf.c | 38 +++++++++++++++------------ utilities/das3_info.c | 28 ++++++++++---------- utilities/das_fetch.c | 60 +++++++++++++++++-------------------------- 11 files changed, 236 insertions(+), 119 deletions(-) diff --git a/das3/catalog.c b/das3/catalog.c index d2bd2e5..27b1c74 100644 --- a/das3/catalog.c +++ b/das3/catalog.c @@ -129,6 +129,17 @@ DasCat* new_DasCat(DasCredMngr* pMgr, const char* sAgent) pThis->pMgr = pMgr; if((sAgent != NULL) && (sAgent[0] != '\0')) pThis->sAgent = das_strdup(sAgent); pThis->sPrefix = das_strdup(DASCAT_DEF_PREFIX); + + /* Caching is on from the start. A session that does not want it says + * so; one that forgets gets fast catalog walks and at worst an hour old + * node, which is the better default for the GUIs and scripts built on + * this. */ + char sDir[sizeof(((DasNodeCache*)0)->sDir)]; + int nLen = snprintf(sDir, sizeof(sDir), "%s" DAS_DSEPS "cache" DAS_DSEPS "nodes", das_userdir()); + if((nLen > 0) && (nLen < (int)sizeof(sDir))) + DasCat_setCache(pThis, sDir, DASCAT_DEF_CACHE_SEC); + else + daslog_warn_v("User directory %s too long for the node cache, caching is off", das_userdir()); return pThis; } @@ -178,10 +189,22 @@ void DasCat_setCache(DasCat* pThis, const char* sDir, int nMaxAgeSec) pThis->pCache->sDir[0] = '\0'; return; } - strcpy(pThis->pCache->sDir, sDir); + /* sDir may be our own buffer, from DasCat_cacheDir(), to change the age */ + if(sDir != pThis->pCache->sDir) strcpy(pThis->pCache->sDir, sDir); pThis->pCache->nMaxAgeSec = nMaxAgeSec; } +const char* DasCat_cacheDir(const DasCat* pThis) +{ + if((pThis->pCache == NULL)||(pThis->pCache->sDir[0] == '\0')) return NULL; + return pThis->pCache->sDir; +} + +int DasCat_cacheAge(const DasCat* pThis) +{ + return (pThis->pCache == NULL) ? 0 : pThis->pCache->nMaxAgeSec; +} + /* ************************************************************************* */ /* Root policy */ diff --git a/das3/catalog.h b/das3/catalog.h index 72033bd..6ee0f1b 100644 --- a/das3/catalog.h +++ b/das3/catalog.h @@ -85,6 +85,9 @@ extern "C" { /** The catalog URI prefix relative paths start from unless told otherwise */ #define DASCAT_DEF_PREFIX "tag:das2.org,2012:site:/" +/** How long, in seconds, a new session trusts a cached catalog node */ +#define DASCAT_DEF_CACHE_SEC 3600 + /** A catalog session, opaque. * * A DasCat turns catalog paths into nodes. It remembers where relative @@ -143,13 +146,16 @@ typedef struct das_cat DasCat; DAS_API bool das_isUrlOrFile(const char* sPath); /** Create a catalog session + * + * The returned catalog it's node cache enabled right out of the gate, + * to turn it off see DasCat_setCache() * * @param pMgr Credentials manager consulted when a node or data source asks * for a login. May be NULL for public access only. Ownership * passes to the catalog, del_DasCat() frees it. * @param sAgent User agent string sent with every request, copied. NULL * sends the library default. - * @returns a new catalog, free with del_DasCat() + * @returns a new catalog, free with del_DasCat(). * @memberof DasCat */ DAS_API DasCat* new_DasCat(DasCredMngr* pMgr, const char* sAgent); @@ -171,21 +177,36 @@ DAS_API DasCredMngr* DasCat_credMngr(const DasCat* pThis); */ DAS_API const char* DasCat_agent(const DasCat* pThis); -/** Keep local copies of catalog nodes this session fetches over HTTP +/** Change where, and for how long, this session keeps copies of catalog + * nodes fetched over HTTP * - * The per session form of DasNode_setCache(). Until called, nodes loaded - * through this catalog follow the process wide setting. Nodes read from - * local files are never cached. + * A new session already caches: copies go under das_userdir() in + * cache/nodes and are trusted for DASCAT_DEF_CACHE_SEC. Walking a catalog + * costs one request per level, so this is what makes the second run of a + * program fast. Nodes read from local files are not cached, and + * neither are nodes made through the constructors in node.h * * @param sDir Directory for the copies, created on first use. NULL or an - * empty string turns caching off for this session, whatever the - * process default says. + * empty string turns caching off for this session. Passing + * DasCat_cacheDir() is allowed, and can be used to change the + * default node expiration time. * @param nMaxAgeSec Copies older than this many seconds are fetched again. - * Zero or less means always fetch, but still save the result. + * Zero or less means always fetch, but still save the result, which + * refreshes the cache along the path walked. * @memberof DasCat */ DAS_API void DasCat_setCache(DasCat* pThis, const char* sDir, int nMaxAgeSec); +/** The directory this session caches nodes in, or NULL if caching is off + * @memberof DasCat + */ +DAS_API const char* DasCat_cacheDir(const DasCat* pThis); + +/** How many seconds this session trusts a cached node, see DasCat_setCache() + * @memberof DasCat + */ +DAS_API int DasCat_cacheAge(const DasCat* pThis); + /** Where relative paths start in the federated catalog * * The shorthands "site:" and "test:" are expanded. A URL or file is diff --git a/das3/node.c b/das3/node.c index 867e135..f283ae1 100644 --- a/das3/node.c +++ b/das3/node.c @@ -311,20 +311,9 @@ static DasAry* _DasNode_readFile(const char* sUrl) /* ************************************************************************* */ /* Node cache */ -/* The process default. A DasCat carries its own settings and nodes it - * loads point there instead, see node_priv.h */ -static DasNodeCache g_defCache = { {'\0'}, 0 }; - -void DasNode_setCache(const char* sDir, int nMaxAgeSec) -{ - if((sDir == NULL)||(sDir[0] == '\0')){ - g_defCache.sDir[0] = '\0'; - return; - } - strncpy(g_defCache.sDir, sDir, sizeof(g_defCache.sDir) - 1); - g_defCache.sDir[sizeof(g_defCache.sDir) - 1] = '\0'; - g_defCache.nMaxAgeSec = nMaxAgeSec; -} +/* Only a DasCat has cache settings, nodes made through the public + * constructors in node.h carry NULL and are never cached. One knob, per + * session, instead of a second process wide one to confuse it with. */ /* Cache file name for a URL: the URL minus its scheme with everything * outside [A-Za-z0-9.-] turned into '_', then a hash of the whole URL so @@ -334,8 +323,7 @@ void DasNode_setCache(const char* sDir, int nMaxAgeSec) static bool _DasNode_cachePath( const DasNodeCache* pCache, const char* sUrl, char* sPath, size_t uLen ){ - if(pCache == NULL) pCache = &g_defCache; - if(pCache->sDir[0] == '\0') return false; + if((pCache == NULL)||(pCache->sDir[0] == '\0')) return false; uint32_t uHash = 2166136261u; /* FNV-1a, no need for anything stronger */ for(const char* p = sUrl; *p != '\0'; ++p){ @@ -362,8 +350,7 @@ static bool _DasNode_cachePath( /* A fresh enough cached copy, or NULL */ static DasAry* _DasNode_readCache(const DasNodeCache* pCache, const char* sPath) { - if(pCache == NULL) pCache = &g_defCache; - if(pCache->nMaxAgeSec <= 0) return NULL; + if((pCache == NULL)||(pCache->nMaxAgeSec <= 0)) return NULL; struct stat st; if(stat(sPath, &st) != 0) return NULL; if((time(NULL) - st.st_mtime) > pCache->nMaxAgeSec) return NULL; diff --git a/das3/node.h b/das3/node.h index 74e300d..8d8b60e 100644 --- a/das3/node.h +++ b/das3/node.h @@ -412,25 +412,6 @@ DAS_API const char* DasNode_label(const DasNode* pThis); */ DAS_API const char* DasNode_pathSep(const DasNode* pThis); -/** Keep local copies of catalog nodes fetched over HTTP. - * - * Walking a catalog from a root to a data source takes one request per - * level. With a cache directory set, each node document is saved after it - * parses and the saved copy is used for later requests of the same URL - * while it is younger than nMaxAgeSec. Nodes read from local files are - * never cached. The setting is process wide. - * - * @param sDir Directory for the copies, created on first use. NULL or an - * empty string turns caching off (the default). - * - * @param nMaxAgeSec Copies older than this many seconds are fetched again. - * Zero or less means always fetch, but still save the result, which - * refreshes the cache along the path walked. - * - * @memberof DasNode - */ -DAS_API void DasNode_setCache(const char* sDir, int nMaxAgeSec); - /** Number of children listed by a container node. * * This inspects the listing only, nothing is downloaded. Children of a diff --git a/das3/node_priv.h b/das3/node_priv.h index 2a5386f..c0956cd 100644 --- a/das3/node_priv.h +++ b/das3/node_priv.h @@ -85,8 +85,9 @@ struct das_node { }; /* Root constructors. The public new_RootNode() and new_RootNode_url() in - * node.h are thin wrappers that pass NULL for pCache and so use the process - * default from DasNode_setCache(). A DasCat passes its own settings. */ + * node.h are thin wrappers that pass NULL for pCache, so trees made through + * them are never cached. A DasCat passes its own settings; the session is + * the only place caching is configured. */ /* Load the node at a catalog path URI by walking down from the built in * federated roots, see das_root_urls(). @@ -111,8 +112,8 @@ struct das_node { * inherited. Borrowed, usually a string literal. * * @param pCache Where fetched node documents are saved and how long they - * are trusted, inherited by the whole tree. NULL selects the process - * default from DasNode_setCache(). Borrowed. + * are trusted, inherited by the whole tree. NULL means no caching. + * Borrowed. * * @returns A new memory root, to be freed with del_RootNode(), or NULL if no * federated root could be reached or none of them lead to sPathUri. @@ -143,7 +144,7 @@ DasNode* _DasNode_newRoot( * @param sAgent The HTTP user agent string, inherited by children. * Borrowed, usually a string literal. * - * @param pCache Cache settings for the tree, NULL for the process default. + * @param pCache Cache settings for the tree, NULL for no caching. * Borrowed. * * @returns A new memory root, to be freed with del_RootNode(), or NULL if @@ -216,7 +217,7 @@ DasNode* _DasNode_newRootUrl( * @param sAgent The HTTP user agent string, inherited by children. * Borrowed, usually a string literal. * - * @param pCache Cache settings for the tree, NULL for the process default. + * @param pCache Cache settings for the tree, NULL for no caching. * Borrowed. * * @returns A new memory root whose source URL reports as "memory:mounts", diff --git a/das3/util.c b/das3/util.c index 8eba615..3536aef 100644 --- a/das3/util.c +++ b/das3/util.c @@ -65,6 +65,7 @@ static das_error_msg* g_msgBuf = NULL; #define HOME_DIR_SZ 256 static char g_sHome[HOME_DIR_SZ] = {'\0'}; +static char g_sUserDir[HOME_DIR_SZ] = {'\0'}; /* Locale handling */ #ifdef _WIN32 @@ -196,6 +197,13 @@ void das_init( else strcpy(g_sHome, "/tmp"); #endif + + /* Everything das keeps for a user lives under one directory, ~/.das on + * every OS the way ssh and git do it, unless DAS_USERDIR says otherwise. + * Read once here so a setenv() later in the process cannot move the + * cache or the credentials file out from under an open session. A + * program that wants to choose calls das_set_userdir(). */ + das_set_userdir(NULL); } void das_finish(){ @@ -716,6 +724,23 @@ const char* das_userhome(void) return g_sHome; } +const char* das_userdir(void) +{ + return g_sUserDir; +} + +void das_set_userdir(const char* sDir) +{ + if((sDir == NULL)||(sDir[0] == '\0')) sDir = getenv("DAS_USERDIR"); + if((sDir != NULL) && (sDir[0] != '\0')){ + strncpy(g_sUserDir, sDir, HOME_DIR_SZ - 1); + g_sUserDir[HOME_DIR_SZ - 1] = '\0'; + } + else{ + snprintf(g_sUserDir, HOME_DIR_SZ, "%s" DAS_DSEPS ".das", g_sHome); + } +} + bool das_isfile(const char* path) { struct stat stbuf; diff --git a/das3/util.h b/das3/util.h index 655d72f..bf2efd3 100644 --- a/das3/util.h +++ b/das3/util.h @@ -503,6 +503,33 @@ DAS_API DasErrCode das_mkdirsto(const char* path); */ DAS_API const char* das_userhome(void); +/** Get the directory for everything das keeps on behalf of the current user + * + * This is ".das" under the home directory on every platform, the way ssh + * and git keep their files, or the value of the DAS_USERDIR environment + * variable if it was set when das_init() ran, or whatever the program gave + * das_setUserDir(). The catalog node cache and saved credentials live + * under it. Only the path is returned, nothing is created; whatever + * writes under it makes the directories it needs. + * + * @return A pointer to a global string, never NULL or empty. + */ +DAS_API const char* das_userdir(void); + +/** Choose the directory das_userdir() reports + * + * For programs that embed the library and want to set the location + * themselves instead of relying on the environment. Call it right after + * das_init() and before any DasCat or other object that reads the + * directory is made: those take their copy when constructed and are not + * moved by a later call. + * + * @param sDir The directory to use, copied. NULL or an empty string goes + * back to the default, the DAS_USERDIR environment variable if set, + * else ".das" under the home directory. + */ +DAS_API void das_set_userdir(const char* sDir); + /** Copy a file to a destination creating directories as needed. * * If the files exists at the destination it in overwritten. Directories are diff --git a/test/TestCat.c b/test/TestCat.c index f37138a..f167c53 100644 --- a/test/TestCat.c +++ b/test/TestCat.c @@ -45,6 +45,10 @@ #define UIOWA "test/das_fetch/cat/uiowa.json" #define UIOWA_URI "tag:das2.org,2012:site:/uiowa" +/* A user directory that does not exist. Nothing writes there: the + fixtures are local files, which are never cached. */ +#define TEST_USERDIR "test_das_userdir_never_created" + static int g_nFail = 0; static int g_nPass = 0; @@ -293,11 +297,67 @@ static void test_mounts(void) del_DasCat(pCat); } +/* ************************************************************************* */ +/* The user directory and the cache default built on it */ + +static bool endsWith(const char* s, const char* sTail) +{ + size_t uS = strlen(s), uT = strlen(sTail); + return (uS >= uT) && (strcmp(s + uS - uT, sTail) == 0); +} + +static void test_userdir(void) +{ + printf("TEST: user directory and the session cache default\n"); + + /* The default, unless this test's own environment set DAS_USERDIR */ + const char* sEnv = getenv("DAS_USERDIR"); + if((sEnv == NULL)||(sEnv[0] == '\0')) + CHECK(endsWith(das_userdir(), DAS_DSEPS ".das"), "default is .das under home, got %s", das_userdir()); + else + CHECK(streq(das_userdir(), sEnv), "DAS_USERDIR is honored, got %s", das_userdir()); + + das_set_userdir(TEST_USERDIR); + CHECK(streq(das_userdir(), TEST_USERDIR), "setter is honored, got %s", das_userdir()); + + DasCat* pCat = new_DasCat(NULL, NULL); + CHECK(streq(DasCat_cacheDir(pCat), TEST_USERDIR DAS_DSEPS "cache" DAS_DSEPS "nodes"), + "new session caches under the user directory, got %s", + DasCat_cacheDir(pCat) ? DasCat_cacheDir(pCat) : "(off)"); + CHECK(DasCat_cacheAge(pCat) == DASCAT_DEF_CACHE_SEC, "default cache age"); + + DasCat_setCache(pCat, DasCat_cacheDir(pCat), 0); /* aliasing is allowed */ + CHECK(streq(DasCat_cacheDir(pCat), TEST_USERDIR DAS_DSEPS "cache" DAS_DSEPS "nodes"), + "age change keeps the directory"); + CHECK(DasCat_cacheAge(pCat) == 0, "age changed to always fetch"); + + DasCat_setCache(pCat, "elsewhere", 5); + CHECK(streq(DasCat_cacheDir(pCat), "elsewhere"), "directory moved"); + CHECK(DasCat_cacheAge(pCat) == 5, "age set with the move"); + + DasCat_setCache(pCat, NULL, 0); + CHECK(DasCat_cacheDir(pCat) == NULL, "NULL turns the cache off"); + DasCat_setCache(pCat, "", 0); + CHECK(DasCat_cacheDir(pCat) == NULL, "empty string turns the cache off"); + del_DasCat(pCat); + + /* A later change does not move a session already made */ + pCat = new_DasCat(NULL, NULL); + das_set_userdir("moved_after_the_fact"); + CHECK(streq(DasCat_cacheDir(pCat), TEST_USERDIR DAS_DSEPS "cache" DAS_DSEPS "nodes"), + "session keeps the directory it was made with"); + del_DasCat(pCat); + + das_set_userdir(NULL); + CHECK(!streq(das_userdir(), "moved_after_the_fact"), "NULL restores the default"); +} + /* ************************************************************************* */ int main(int argc, char** argv) { das_init(argv[0], DASERR_DIS_EXIT, 0, DASLOG_WARN, NULL); + test_userdir(); test_isUrlOrFile(); test_policy(); test_rootMode(); diff --git a/utilities/das3_cdf.c b/utilities/das3_cdf.c index 3e5b551..11244b0 100644 --- a/utilities/das3_cdf.c +++ b/utilities/das3_cdf.c @@ -60,8 +60,8 @@ #define DASCDF_COMP_STR_SZ 32 /* room for one component label or units string */ #define DASCDF_MAX_ATTR_MAP 16 /* --prop-map entries */ -#define DEF_AUTH_FILE ".dasauth" -#define DEF_TEMP_DIR ".dastmp" +#define CRED_FILE "servers" /* under das_userdir() */ +#define DEF_TEMP_DIR "cdftmp" /* under das_userdir(), so one rm -r clears it */ #define LOC_PATH_LEN 256 @@ -69,13 +69,13 @@ #define VAR_MAP_MAX_LINE 256 +/* For help text only, das_userdir() does the lookup */ #ifdef _WIN32 -#define HOME_VAR_STR "USERPROFILE" -#define HELP_TEMP_DIR "%USERPROFILE%\\" DEF_TEMP_DIR +#define HELP_USERDIR "%USERPROFILE%\\.das" #else -#define HOME_VAR_STR "HOME" -#define HELP_TEMP_DIR "$HOME/" DEF_TEMP_DIR +#define HELP_USERDIR "$HOME/.das" #endif +#define HELP_TEMP_DIR HELP_USERDIR DAS_DSEPS DEF_TEMP_DIR /* Add a little user-flag for arrays so we know which ones to clear after a batch write */ @@ -253,11 +253,15 @@ void prnHelp() "\n" " -t DIR,--temp-dir=DIR\n" " Directory for writing temporary files when run as a command\n" -" pipeline filter. Defaults to \"%s\". Ignored if -o is given.\n" +" pipeline filter. Defaults to \"%s\", or the same\n" +" place under DAS_USERDIR if that environment variable is set.\n" +" Use this, or DAS_USERDIR, when running under a web server\n" +" whose account has no writable home. Ignored if -o is given.\n" "\n" " -a FILE,--auth-toks=FILE\n" -" Set the location where server authentication tokens (if any)\n" -" are saved. Defaults to %s%s%s\n" +" Read saved server credentials from FILE. Defaults to\n" +" " HELP_USERDIR DAS_DSEPS CRED_FILE ", or the same file under DAS_USERDIR if that\n" +" environment variable is set.\n" "\n" " -i URL,--input=URL\n" " Instead of reading from standard input, read from a given URL.\n" @@ -338,7 +342,7 @@ void prnHelp() " This is needed for any CDF files submitted to the Planetary Data\n" " system. Per ISTP rules, Epoch variables are not compressed.\n" "\n", -HELP_TEMP_DIR, HOME_VAR_STR, DAS_DSEPS, DEF_AUTH_FILE); +HELP_TEMP_DIR); printf( @@ -424,15 +428,15 @@ int parseArgs(int argc, char** argv, popts_t* pOpts) /* Set a few defaults */ snprintf( - pOpts->aCredFile, DAS_FIELD_SZ(popts_t, aCredFile) - 1, "%s" DAS_DSEPS DEF_AUTH_FILE, - das_userhome() + pOpts->aCredFile, DAS_FIELD_SZ(popts_t, aCredFile) - 1, "%s" DAS_DSEPS CRED_FILE, + das_userdir() ); strcpy(pOpts->aLevel, "info"); snprintf( - pOpts->aTmpDir, DAS_FIELD_SZ(popts_t, aTmpDir) - 1, "%s" DAS_DSEPS ".cdftmp", - das_userhome() + pOpts->aTmpDir, DAS_FIELD_SZ(popts_t, aTmpDir) - 1, "%s" DAS_DSEPS DEF_TEMP_DIR, + das_userdir() ); int i = 0; @@ -3568,10 +3572,11 @@ int main(int argc, char** argv) else{ /* Just writing to a temporary location */ /* Temporary write and stream. Would love to use tmpfile here, but - libcdf doesn't support it */ + libcdf doesn't support it. An explicit -t "" falls back to the + default, parseArgs() fills it otherwise. */ if(opts.aTmpDir[0] == '\0'){ snprintf(ctx.sWriteTo, LOC_PATH_LEN-1, "%s%c%s%cd3cdf-tmp-%d.cdf", - das_userhome(), DAS_DSEPC, DEF_TEMP_DIR, DAS_DSEPC, getpid() + das_userdir(), DAS_DSEPC, DEF_TEMP_DIR, DAS_DSEPC, getpid() ); } else{ @@ -3609,6 +3614,7 @@ int main(int argc, char** argv) ){ pCreds = new_CredMngr(opts.aCredFile); + if(das_isfile(opts.aCredFile)) CredMngr_load(pCreds, NULL, NULL); /* Give it a connection time out of 6 seconds */ if(!das_http_getBody(opts.aSource, "das3_cdf", pCreds, &res, 6.0)){ diff --git a/utilities/das3_info.c b/utilities/das3_info.c index 3df0e9e..05df15e 100644 --- a/utilities/das3_info.c +++ b/utilities/das3_info.c @@ -42,15 +42,13 @@ /* Globals */ #define PERR DASERR_MAX + 1 -#define DEF_AUTH_FILE ".dasauth" +#define CRED_FILE "servers" /* under das_userdir() */ #define PROG "das3_info" #ifndef _WIN32 -#define HOME_VAR "$HOME" -#define HOME_ENV "HOME" +#define HELP_USERDIR "$HOME/.das" /* for help text, das_userdir() does the lookup */ #else -#define HOME_VAR "%%USERPROFILE%%" -#define HOME_ENV "USERPROFILE" +#define HELP_USERDIR "%%USERPROFILE%%\\.das" #endif /* ************************************************************************* */ @@ -92,8 +90,9 @@ void prnHelp() " critical. The default is warning.\n" "\n" "AUTHENTICATION\n" -" URLs that require a login are handled by prompting for credentials,\n" -" which are cached for the session in " HOME_VAR "/" DEF_AUTH_FILE ".\n" +" URLs that require a login prompt for a username and password. Saved\n" +" credentials in " HELP_USERDIR DAS_DSEPS CRED_FILE " are tried first. The environment\n" +" variable DAS_USERDIR replaces " HELP_USERDIR ".\n" "\n" "EXAMPLES\n" " Describe the Cassini/RPWS waveform example provided with the source:\n" @@ -107,7 +106,8 @@ void prnHelp() "SEE ALSO\n" " das3_text, das3_csv\n" "\n" -" The das3 data model is described at https://das2.org/das3\n" +" The das3 data model and this library are documented at\n" +" https://das-developers.github.io/das2C/\n" "\n" ); } @@ -122,15 +122,13 @@ static DasIO* openInput(const char* sInput, float rTimeout) if((strncmp(sInput, "http://", 7) != 0)&&(strncmp(sInput, "https://", 8) != 0)) return new_DasIO_file(PROG, sInput, "r"); - /* A URL: the credential manager only matters if the server asks */ - char* sHome = getenv(HOME_ENV); - if(sHome == NULL){ - das_error(PERR, "Environment variable %s not set", HOME_ENV); - return NULL; - } + /* A URL: the credential manager only matters if the server asks. Saved + credentials are loaded only if the file exists, since CredMngr_load + logs an error for a missing one and most users have none. */ char sCredFile[256] = {'\0'}; - snprintf(sCredFile, 255, "%s%s%s", sHome, DAS_DSEPS, DEF_AUTH_FILE); + snprintf(sCredFile, sizeof(sCredFile), "%s" DAS_DSEPS CRED_FILE, das_userdir()); DasCredMngr* pCreds = new_CredMngr(sCredFile); + if(das_isfile(sCredFile)) CredMngr_load(pCreds, NULL, NULL); DasHttpResp res; memset(&res, 0, sizeof(DasHttpResp)); diff --git a/utilities/das_fetch.c b/utilities/das_fetch.c index 0f2965a..b2b5ae6 100644 --- a/utilities/das_fetch.c +++ b/utilities/das_fetch.c @@ -67,11 +67,10 @@ #define PROG "das_fetch" #define PERR (DASERR_MAX + 1) -#define DAS_DIR ".das" /* under the home directory, holds cache/ and servers */ -#define CRED_FILE "servers" /* under DAS_DIR; the older ~/.dasauth is not read */ +#define CRED_FILE "servers" /* under das_userdir(); the older ~/.dasauth is not read */ #ifndef _WIN32 -#define HOME_VAR "$HOME" /* for help text, das_userhome() does the lookup */ +#define HOME_VAR "$HOME" /* for help text, das_userdir() does the lookup */ #else #define HOME_VAR "%USERPROFILE%" #endif @@ -99,7 +98,6 @@ #define URL_SZ 4096 #define ERR_SZ 1024 -#define DEF_CACHE_AGE 3600 /* seconds a cached catalog node is trusted */ #define ROOT_ENV "DASFETCH_ROOT" #define MOUNT_ENV "DASFETCH_MOUNT" @@ -200,13 +198,14 @@ static void prnHelpMain(void) " -l LEVEL Log level: crit, error, warn, info, debug, trace.\n" "\n" "FILES\n" -" " HOME_VAR "/" DAS_DIR "/cache/nodes/\n" +" " HOME_VAR "/.das/cache/nodes/\n" " Catalog nodes fetched over HTTP, kept for an hour. Walking a catalog\n" " costs one request per level, so repeat runs only talk to the data\n" " server. See -f and -C.\n" -" " HOME_VAR "/" DAS_DIR "/" CRED_FILE "\n" +" " HOME_VAR "/.das/" CRED_FILE "\n" " Saved credentials. When a source needs them these are tried before\n" " the username and password prompt.\n" +" The environment variable DAS_USERDIR replaces " HOME_VAR "/.das for both.\n" "\n" "EXIT CODES\n" " 0 success 2 usage error 3 path did not resolve 4 bad query\n" @@ -1928,17 +1927,6 @@ static bool parseShow(const char* sList, unsigned* puShow) return true; } -/* The default cache directory. Everything das keeps for a user lives under - * one dot directory in the home directory on every OS, the way ssh, aws and - * friends do it, so the documentation names one place instead of three. */ -static void defCacheDir(char* sBuf, size_t uLen) -{ - sBuf[0] = '\0'; - const char* sHome = das_userhome(); - if(sHome && sHome[0]) - snprintf(sBuf, uLen, "%s" DAS_DSEPS DAS_DIR DAS_DSEPS "cache" DAS_DSEPS "nodes", sHome); -} - int main(int argc, char** argv) { das_init(PROG, DASERR_DIS_EXIT, 0, DASLOG_WARN, NULL); @@ -2035,19 +2023,16 @@ int main(int argc, char** argv) /* Credentials: saved ones are tried before the terminal prompt. The catalog owns the manager and hands it to every node and request. */ char sCredFile[256] = {'\0'}; - const char* sHome = das_userhome(); - if(sHome && sHome[0]) - snprintf(sCredFile, sizeof(sCredFile), "%s" DAS_DSEPS DAS_DIR DAS_DSEPS CRED_FILE, sHome); - DasCredMngr* pMgr = new_CredMngr(sCredFile[0] ? sCredFile : NULL); - if(sCredFile[0]){ - /* CredMngr_load logs an error for a missing file even though its - contract says a missing file is not an error. Most users have no - saved credentials, so look first. */ - FILE* pTest = fopen(sCredFile, "r"); - if(pTest != NULL){ - fclose(pTest); - CredMngr_load(pMgr, NULL, NULL); - } + snprintf(sCredFile, sizeof(sCredFile), "%s" DAS_DSEPS CRED_FILE, das_userdir()); + DasCredMngr* pMgr = new_CredMngr(sCredFile); + + /* CredMngr_load logs an error for a missing file even though its + contract says a missing file is not an error. Most users have no + saved credentials, so look first. */ + FILE* pTest = fopen(sCredFile, "r"); + if(pTest != NULL){ + fclose(pTest); + CredMngr_load(pMgr, NULL, NULL); } pCat = new_DasCat(pMgr, sAgent); @@ -2090,12 +2075,15 @@ int main(int argc, char** argv) if(bNode && (uKeyVals > 0)) daslog_warn("KEY=VALUE arguments are ignored by the node command"); - /* Node cache, unless told otherwise. -f refreshes rather than disables - so that the run after a catalog change leaves the cache current. */ - char sCacheDir[512] = {'\0'}; - if(sCacheArg == NULL) defCacheDir(sCacheDir, sizeof(sCacheDir)); - else if(!strieq(sCacheArg, "none")) strcpyz(sCacheDir, sCacheArg, sizeof(sCacheDir)); - DasCat_setCache(pCat, sCacheDir, bFresh ? 0 : DEF_CACHE_AGE); /* empty dir: off */ + /* Node cache: the session's default unless told otherwise. -f refreshes + rather than disables so that the run after a catalog change leaves the + cache current. */ + if((sCacheArg != NULL) && strieq(sCacheArg, "none")) + DasCat_setCache(pCat, NULL, 0); + else if(sCacheArg != NULL) + DasCat_setCache(pCat, sCacheArg, bFresh ? 0 : DASCAT_DEF_CACHE_SEC); + else if(bFresh) + DasCat_setCache(pCat, DasCat_cacheDir(pCat), 0); /* A fragment on the path is only meaningful for the json command */ char sPathBuf[1024] = {'\0'}; From b7b9937f20170c0b66c6447e734e90ed611bc04a Mon Sep 17 00:00:00 2001 From: cpiker Date: Mon, 26 Oct 2026 03:21:19 -0500 Subject: [PATCH 06/18] catalog: aged cache copies are revalidated, not refetched A cached node keeps the ETag and Last-Modified it came with. Past the trust age the session asks the server with If-None-Match and If-Modified-Since, and a 304 renews the copy for no body bytes. An unreachable server leaves the old copy in use with a warning. http.h gains das_http_getBody_ex and das_http_readUrl_ex for conditional requests and DasHttpResp_header for reading response headers. Developed-with: Claude Fable 5.1 --- buildfiles/Darwin.mak | 1 + buildfiles/Linux.mak | 1 + buildfiles/cmake/das2CTests.cmake | 3 +- das3/catalog.h | 15 ++-- das3/http.c | 75 +++++++++++++++++- das3/http.h | 58 ++++++++++++++ das3/node.c | 123 ++++++++++++++++++++++++++---- das3/node_priv.h | 2 +- test/TestHttp.c | 31 +++++++- test/das_cache_srv.py | 107 ++++++++++++++++++++++++++ test/das_cache_test.sh | 122 +++++++++++++++++++++++++++++ 11 files changed, 510 insertions(+), 28 deletions(-) create mode 100755 test/das_cache_srv.py create mode 100755 test/das_cache_test.sh diff --git a/buildfiles/Darwin.mak b/buildfiles/Darwin.mak index 85dd4fc..c59b91f 100644 --- a/buildfiles/Darwin.mak +++ b/buildfiles/Darwin.mak @@ -402,6 +402,7 @@ test_main: $(BD) $(BD)/$(TARG).a $(BUILD_TEST_PROGS) $(BULID_UTIL_PROGS) test/das3_info_test.sh $(BD) test/das3_from_das2_test.sh $(BD) test/das_fetch_test.sh $(BD) # test/das_fetch/*.json nodes -> query URLs, offline + test/das_cache_test.sh $(BD) # node cache revalidation against a local python server @echo "INFO: Running unit test for CSVs with variable length item data, $(BD)/das3_csv" $(BD)/das3_csv < examples/ex21_tracers_cdpu_status.d3b > $(BD)/ex21_tracers_cdpu_status.csv @echo "INFO: ===============================================" diff --git a/buildfiles/Linux.mak b/buildfiles/Linux.mak index 80e857a..a0b0893 100644 --- a/buildfiles/Linux.mak +++ b/buildfiles/Linux.mak @@ -402,6 +402,7 @@ test_main: $(BD) $(BD)/$(TARG).a $(BUILD_TEST_PROGS) $(BUILD_UTIL_PROGS) test/das3_info_test.sh $(BD) # examples/ex*.info golds for the dataset printers test/das3_from_das2_test.sh $(BD) # examples/ex11 pass-through, metafile gold, reject test/das_fetch_test.sh $(BD) # test/das_fetch/*.json nodes -> query URLs, offline + test/das_cache_test.sh $(BD) # node cache revalidation against a local python server @echo "INFO: Running unit test for CSVs with variable length item data, $(BD)/das3_csv" $(BD)/das3_csv < examples/ex21_tracers_cdpu_status.d3b > $(BD)/ex21_tracers_cdpu_status.csv @echo "INFO: ===============================================" diff --git a/buildfiles/cmake/das2CTests.cmake b/buildfiles/cmake/das2CTests.cmake index 3a78692..717c917 100644 --- a/buildfiles/cmake/das2CTests.cmake +++ b/buildfiles/cmake/das2CTests.cmake @@ -86,7 +86,8 @@ _das_redir(das3_csv_varlen main ) # das_fetch: a saved catalog node and its example 0 become a query URL, no -# network. The full offline suite is test/das_fetch_test.sh (bash only). +# network. The full offline suite is test/das_fetch_test.sh (bash only), and +# test/das_cache_test.sh covers node cache revalidation (bash and python3). _das_test(das_fetch_url main das_fetch -C none url test/das_fetch/geomag_flex_0.7.json 0) set_tests_properties(das_fetch_url PROPERTIES PASS_REGULAR_EXPRESSION "read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" diff --git a/das3/catalog.h b/das3/catalog.h index 6ee0f1b..4dac9c1 100644 --- a/das3/catalog.h +++ b/das3/catalog.h @@ -183,16 +183,21 @@ DAS_API const char* DasCat_agent(const DasCat* pThis); * A new session already caches: copies go under das_userdir() in * cache/nodes and are trusted for DASCAT_DEF_CACHE_SEC. Walking a catalog * costs one request per level, so this is what makes the second run of a - * program fast. Nodes read from local files are not cached, and - * neither are nodes made through the constructors in node.h + * program fast. A copy older than that is not thrown away: the server is + * asked whether it changed, using the ETag or Last-Modified it came with, + * and only a changed node is downloaded again. A server that cannot be + * reached leaves the old copy in use, with a warning. Nodes read from + * local files are not cached, and neither are nodes made through the + * constructors in node.h * * @param sDir Directory for the copies, created on first use. NULL or an * empty string turns caching off for this session. Passing * DasCat_cacheDir() is allowed, and can be used to change the * default node expiration time. - * @param nMaxAgeSec Copies older than this many seconds are fetched again. - * Zero or less means always fetch, but still save the result, which - * refreshes the cache along the path walked. + * @param nMaxAgeSec Copies older than this many seconds are checked with + * the server before use. Zero or less means check every time, + * which costs one round trip per level and no body bytes when the + * server supports validators, and a full fetch when it does not. * @memberof DasCat */ DAS_API void DasCat_setCache(DasCat* pThis, const char* sDir, int nMaxAgeSec); diff --git a/das3/http.c b/das3/http.c index bb29d92..6be35fc 100644 --- a/das3/http.c +++ b/das3/http.c @@ -53,6 +53,12 @@ #include "info_priv.h" #include "util_priv.h" +#ifdef _WIN32 +#define strncasecmp _strnicmp +#else +#include +#endif + #ifdef _WIN32 typedef ptrdiff_t ssize_t; #endif @@ -81,6 +87,7 @@ char html5[256] = { #define LIBDAS_USER_AGENT "libdas3" #define HTTP_OK 200 +#define HTTP_NotModified 304 #define HTTP_MovedPerm 301 #define HTTP_Found 302 #define HTTP_TempRedir 307 /* Treat as 307 */ @@ -724,7 +731,8 @@ static bool _das_http_connect(DasHttpResp* pRes, struct timeval* pTimeOut) static bool _das_http_getRequest( - DasHttpResp* pRes, const char* sAgent, const char* sAuth, DasBuf* pBuf + DasHttpResp* pRes, const char* sAgent, const char* sAuth, const char* sReqHdrs, + DasBuf* pBuf ){ /* Have to handle the fact that blocking SIGPIPE is different on Apple */ #if defined(__APPLE__) || defined(_WIN32) || defined(__sun) @@ -749,6 +757,8 @@ static bool _das_http_getRequest( /* fprintf(stderr, "Authorization: Basic %s\r\n", sAuth); */ DasBuf_printf(pBuf, "Authorization: Basic %s\r\n", sAuth); } + if((sReqHdrs != NULL)&&(sReqHdrs[0] != '\0')) + DasBuf_printf(pBuf, "%s", sReqHdrs); /* caller's lines, each CRLF ended */ DasBuf_printf(pBuf, "Connection: close\r\n\r\n"); size_t uSent = 0; @@ -834,6 +844,38 @@ static bool _das_http_hdrSearch(DasBuf* pBuf, const char* sField, char* sBuf, si return false; } +bool DasHttpResp_header( + const DasHttpResp* pRes, const char* sField, char* sBuf, size_t uLen +){ + if(uLen > 0) sBuf[0] = '\0'; + if((pRes == NULL)||(pRes->sHeaders == NULL)||(sField == NULL)||(uLen == 0)) + return false; + + /* Field names are case insensitive (RFC 9110), and servers do differ: + Apache sends ETag, GitHub sends etag. Values are trimmed. */ + size_t uField = strlen(sField); + const char* pLine = pRes->sHeaders; + while(*pLine != '\0'){ + const char* pEnd = strstr(pLine, "\r\n"); + size_t uLine = pEnd ? (size_t)(pEnd - pLine) : strlen(pLine); + if((uLine > uField) && (pLine[uField] == ':') && + (strncasecmp(pLine, sField, uField) == 0)){ + const char* pVal = pLine + uField + 1; + const char* pValEnd = pLine + uLine; + while((pVal < pValEnd) && isspace((unsigned char)*pVal)) ++pVal; + while((pValEnd > pVal) && isspace((unsigned char)pValEnd[-1])) --pValEnd; + size_t uVal = (size_t)(pValEnd - pVal); + if(uVal >= uLen) return false; /* does not fit, never truncated */ + memcpy(sBuf, pVal, uVal); + sBuf[uVal] = '\0'; + return true; + } + if(pEnd == NULL) break; + pLine = pEnd + 2; + } + return false; +} + static bool _das_http_setFileName(DasBuf* pBuf, DasHttpResp* pRes) { char sDis[196] = {'\0'}; @@ -1022,9 +1064,18 @@ void DasHttpResp_freeFields(DasHttpResp* pRes){ /* ************************************************************************* */ /* Get a message body socket, involves quite a few steps */ +static void _das_http_closeBody(DasHttpResp* pRes); + bool das_http_getBody( const char* sUrl, const char* sAgent, DasCredMngr* pMgr, DasHttpResp* pRes, float rConSec +){ + return das_http_getBody_ex(sUrl, sAgent, pMgr, pRes, rConSec, NULL); +} + +bool das_http_getBody_ex( + const char* sUrl, const char* sAgent, DasCredMngr* pMgr, DasHttpResp* pRes, + float rConSec, const char* sReqHdrs ){ DasHttpResp_clear(pRes); /* Sets nSockFd to -1 */ @@ -1051,7 +1102,7 @@ bool das_http_getBody( } /* Send the request */ - if(! _das_http_getRequest(pRes, sAgent, sAuth, pBuf) ){ + if(! _das_http_getRequest(pRes, sAgent, sAuth, sReqHdrs, pBuf) ){ goto CLEANUP_ERROR; } @@ -1078,6 +1129,16 @@ bool das_http_getBody( del_DasBuf(pBuf); return true; break; + case HTTP_NotModified: + /* Only reachable when sReqHdrs carried a validator. There is no + body, so the transport is released here and the caller has the + headers and nCode to go on. */ + pRes->sHeaders = (char*)calloc(DasBuf_written(pBuf) + 1, sizeof(char)); + DasBuf_read(pBuf, pRes->sHeaders, DasBuf_written(pBuf)); + del_DasBuf(pBuf); + _das_http_closeBody(pRes); + return true; + break; case HTTP_MovedPerm: case HTTP_Found: case HTTP_TempRedir: @@ -1249,9 +1310,17 @@ DasAry* das_http_readUrl( const char* sUrl, const char* sAgent, DasCredMngr* pMgr, DasHttpResp* pRes, int64_t nLimit, float rConSec ){ - if( ! das_http_getBody(sUrl, sAgent, pMgr, pRes, rConSec)){ + return das_http_readUrl_ex(sUrl, sAgent, pMgr, pRes, nLimit, rConSec, NULL); +} + +DasAry* das_http_readUrl_ex( + const char* sUrl, const char* sAgent, DasCredMngr* pMgr, DasHttpResp* pRes, + int64_t nLimit, float rConSec, const char* sReqHdrs +){ + if( ! das_http_getBody_ex(sUrl, sAgent, pMgr, pRes, rConSec, sReqHdrs)){ return NULL; } + if(pRes->nCode == HTTP_NotModified) return NULL; /* no body by design, see http.h */ if(nLimit < 1) nLimit = -1; int64_t nTotal = 0; diff --git a/das3/http.h b/das3/http.h index 44d581d..435d727 100644 --- a/das3/http.h +++ b/das3/http.h @@ -164,6 +164,24 @@ DAS_API bool DasHttpResp_init(DasHttpResp* pRes, const char* sUrl); */ DAS_API bool DasHttpResp_useSsl(DasHttpResp* pRes); +/** Get the value of one header from the final response + * + * Field names match without regard to case, as HTTP requires. The value + * is copied with surrounding white space removed. Only the headers of the + * final response are searched, those of any redirect on the way are gone. + * + * @param sField The header name, for example "ETag" or "Last-Modified" + * @param sBuf Receives the value, always NUL terminated when uLen > 0 + * @param uLen Size of sBuf including the terminator + * @returns true if the header is present and its value fit, false if it is + * absent, the response has no headers yet, or the value would + * have been truncated + * @memberof DasHttpResp + */ +DAS_API bool DasHttpResp_header( + const DasHttpResp* pRes, const char* sField, char* sBuf, size_t uLen +); + /** Get a socket positioned at the start of a remote resource * * This function makes a connection to a remote HTTP or HTTPS server, reads @@ -224,6 +242,34 @@ DAS_API bool das_http_getBody( const char* sUrl, const char* sAgent, DasCredMngr* pMgr, DasHttpResp* pRes, float rConSec ); + +/** das_http_getBody() with extra request headers, for conditional requests + * + * The one use today is cache revalidation. A caller holding a copy of a + * resource along with the ETag or Last-Modified header it came with sends + * + * "If-None-Match: \"5a6-607e10c29b347\"\r\nIf-Modified-Since: Tue, 17 Oct 2023 03:23:33 GMT\r\n" + * + * and a server that still has that version answers 304 Not Modified with + * no body. Send both validators when both are known: a server with an + * ETag ignores If-Modified-Since (RFC 9110) and one without an ETag can + * only use the date. Values go back exactly as the server sent them, + * see DasHttpResp_header(). + * + * @param sReqHdrs Zero or more complete header lines, each ending in + * "\r\n", appended to the request as given. NULL or empty sends a + * plain request, which is what das_http_getBody() does. + * + * @returns As das_http_getBody(), plus one case: true with pRes->nCode + * 304 and no body socket, nSockFd -1 and pSsl NULL, when the + * server reports the copy is current. pRes->sHeaders holds the + * 304 response's headers. Callers that sent no validator never + * see this. + */ +DAS_API bool das_http_getBody_ex( + const char* sUrl, const char* sAgent, DasCredMngr* pMgr, DasHttpResp* pRes, + float rConSec, const char* sReqHdrs +); /** Read all the bytes for a URL into a byte array * @@ -271,6 +317,18 @@ DAS_API DasAry* das_http_readUrl( int64_t nLimit, float rConSec ); +/** das_http_readUrl() with extra request headers, see das_http_getBody_ex() + * + * @returns The body as das_http_readUrl() does, or NULL in two cases the + * caller tells apart by pRes->nCode: 304, the server says the + * caller's copy is current and sent no body, pRes->sError is NULL; + * anything else, the request failed and pRes->sError says why. + */ +DAS_API DasAry* das_http_readUrl_ex( + const char* sUrl, const char* sAgent, DasCredMngr* pMgr, DasHttpResp* pRes, + int64_t nLimit, float rConSec, const char* sReqHdrs +); + /** Progress callback for das_http_copyBody() * * Follows the StreamHandler convention: a non-zero return stops the copy diff --git a/das3/node.c b/das3/node.c index f283ae1..a2b7bbf 100644 --- a/das3/node.c +++ b/das3/node.c @@ -382,6 +382,68 @@ static void _DasNode_writeCache(const char* sPath, const char* pBytes, size_t uB } } +/* The validators a cached node came with live beside it in a .hdr side + * file, at most two lines, the ETag and Last-Modified header values + * verbatim. When the copy is older than the session trusts both go back + * as If-None-Match and If-Modified-Since, and a 304 renews the copy for + * one round trip and no body. Both are kept because servers differ: the + * das2.org tree sends both, GitHub raw sends ETag only, dasflex today + * sends neither. The date is never derived from the cache file's own + * mtime, which renewal moves forward. */ +static void _DasNode_hdrPath(const char* sCache, char* sHdr, size_t uLen) +{ + snprintf(sHdr, uLen, "%s.hdr", sCache); +} + +/* Request header lines for the cached copy at sCache, "" if it has none */ +static void _DasNode_readValidators(const char* sCache, char* sReq, size_t uLen) +{ + sReq[0] = '\0'; + char sHdr[600]; + _DasNode_hdrPath(sCache, sHdr, sizeof(sHdr)); + FILE* pFile = fopen(sHdr, "r"); + if(pFile == NULL) return; + + char sLine[1024]; + size_t u = 0; + while(fgets(sLine, sizeof(sLine), pFile) != NULL){ + size_t uLine = strlen(sLine); + while((uLine > 0) && ((sLine[uLine-1] == '\n')||(sLine[uLine-1] == '\r'))) + sLine[--uLine] = '\0'; + const char* sName = NULL; + const char* sVal = NULL; + if(strncmp(sLine, "ETag: ", 6) == 0){ sName = "If-None-Match"; sVal = sLine + 6; } + else if(strncmp(sLine, "Last-Modified: ", 15) == 0){ sName = "If-Modified-Since"; sVal = sLine + 15; } + if((sName == NULL)||(sVal[0] == '\0')) continue; + int n = snprintf(sReq + u, uLen - u, "%s: %s\r\n", sName, sVal); + if((n < 0)||((size_t)n >= uLen - u)){ sReq[u] = '\0'; break; } + u += (size_t)n; + } + fclose(pFile); +} + +/* Save the validators a 200 carried, or drop the side file if it carried + * none, so a validator from an older body is never sent for a newer one */ +static void _DasNode_writeValidators(const char* sCache, const DasHttpResp* pRes) +{ + char sHdr[600]; + _DasNode_hdrPath(sCache, sHdr, sizeof(sHdr)); + char sETag[256] = {'\0'}; + char sMod[128] = {'\0'}; + bool bETag = DasHttpResp_header(pRes, "ETag", sETag, sizeof(sETag)); + bool bMod = DasHttpResp_header(pRes, "Last-Modified", sMod, sizeof(sMod)); + if(!bETag && !bMod){ remove(sHdr); return; } + + FILE* pFile = fopen(sHdr, "w"); + if(pFile == NULL){ + daslog_debug_v("Couldn't write node cache side file %s, %s", sHdr, strerror(errno)); + return; + } + if(bETag) fprintf(pFile, "ETag: %s\n", sETag); + if(bMod) fprintf(pFile, "Last-Modified: %s\n", sMod); + fclose(pFile); +} + /* ************************************************************************* */ /* Direct Node Construction */ @@ -507,7 +569,11 @@ static DasNode* _DasNode_mkNode( DasAry* pBytesAry = NULL; char sCache[512] = {'\0'}; - bool bFromCache = false; + bool bFromCache = false; /* copy used as is, nothing to write back */ + bool bRenewed = false; /* server confirmed the copy, rewrite it to move its age */ + DasHttpResp httpRes; + memset(&httpRes, 0, sizeof(DasHttpResp)); + if(_DasNode_isFileUrl(sUrl)){ pBytesAry = _DasNode_readFile(sUrl); strncpy(sFinalUrl, sUrl, sizeof(sFinalUrl) - 1); @@ -519,32 +585,57 @@ static DasNode* _DasNode_mkNode( bFromCache = true; } else{ - /* Read JSON bytes into the array, put a sanity limit of 20 MB on the - * read, no catalogs should ever get anywhere near that size, but you - * don't know */ - DasHttpResp httpRes; - pBytesAry = das_http_readUrl( - sUrl, sAgent, pMgr, &httpRes, 1024*1024*20, rConSec + /* No copy, or one older than the session trusts. With validators + * from the old copy the request is conditional and a 304 renews it + * for no body bytes. If the server cannot be reached at all the old + * copy still beats nothing: a warm cache keeps working offline. The + * 20 MB limit is a sanity check, no catalog node comes near it. */ + char sReqHdrs[1024] = {'\0'}; + bool bHaveOld = (sCache[0] != '\0') && das_isfile(sCache); + if(bHaveOld) _DasNode_readValidators(sCache, sReqHdrs, sizeof(sReqHdrs)); + + pBytesAry = das_http_readUrl_ex( + sUrl, sAgent, pMgr, &httpRes, 1024*1024*20, rConSec, sReqHdrs ); - if(pBytesAry == NULL){ - daslog_warn_v("%s", httpRes.sError); - DasHttpResp_freeFields(&httpRes); - return NULL; + if(pBytesAry != NULL){ + das_url_toStr(&(httpRes.url), sFinalUrl, sizeof(sFinalUrl)); + } + else if(httpRes.nCode == 304){ + daslog_debug_v("Node %s unchanged on the server, cached copy renewed", sUrl); + pBytesAry = _DasNode_readFile(sCache); + strncpy(sFinalUrl, sUrl, sizeof(sFinalUrl) - 1); + bRenewed = true; + } + else if(bHaveOld){ + daslog_warn_v("%s, using the cached copy of %s", + httpRes.sError ? httpRes.sError : "Fetch failed", sUrl); + pBytesAry = _DasNode_readFile(sCache); + strncpy(sFinalUrl, sUrl, sizeof(sFinalUrl) - 1); + bFromCache = true; + } + else{ + daslog_warn_v("%s", httpRes.sError ? httpRes.sError : "Fetch failed"); } - das_url_toStr(&(httpRes.url), sFinalUrl, sizeof(sFinalUrl)); - DasHttpResp_freeFields(&httpRes); /* Done with the headers etc. */ } - if(pBytesAry == NULL) return NULL; + if(pBytesAry == NULL){ + DasHttpResp_freeFields(&httpRes); + return NULL; + } size_t uBytes; const char* pBytes = (const char*) (DasAry_getBytesIn(pBytesAry, DIM0, &uBytes)); DasNode* pNode = _DasNode_fromJson(pBytes, uBytes, sUrl, sFinalUrl, sPathUri); if(pNode != NULL){ pNode->pMgr = pMgr; pNode->sAgent = sAgent; pNode->pCache = pCache; } - /* Only documents that parse are worth keeping, a server error page is not */ - if((pNode != NULL) && (sCache[0] != '\0') && !bFromCache) + /* Only documents that parse are worth keeping, a server error page is + * not. A renewal rewrites the same bytes, which is what moves the + * copy's age, and keeps the validators it already has. */ + if((pNode != NULL) && (sCache[0] != '\0') && !bFromCache){ _DasNode_writeCache(sCache, pBytes, uBytes); + if(!bRenewed) _DasNode_writeValidators(sCache, &httpRes); + } + DasHttpResp_freeFields(&httpRes); dec_DasAry(pBytesAry); /* Done with the byte array, data in json buffer now */ return pNode; } diff --git a/das3/node_priv.h b/das3/node_priv.h index c0956cd..2f18fe9 100644 --- a/das3/node_priv.h +++ b/das3/node_priv.h @@ -70,7 +70,7 @@ */ typedef struct das_node_cache { char sDir[256]; /* empty means no caching */ - int nMaxAgeSec; /* zero or less means always fetch, but still save */ + int nMaxAgeSec; /* zero or less means always ask the server, see _DasNode_mkNode() */ } DasNodeCache; struct das_node { diff --git a/test/TestHttp.c b/test/TestHttp.c index 433a85d..0d5d9fb 100644 --- a/test/TestHttp.c +++ b/test/TestHttp.c @@ -1,5 +1,5 @@ -/** @file TestHttp.c Unit tests for http.h that need no network: today the - * URL parser behind DasHttpResp_init(). */ +/** @file TestHttp.c Unit tests for http.h that need no network: the URL + * parser behind DasHttpResp_init() and the response header lookup. */ /* Author: Chris Piker , via Claude Fable 5.1 * @@ -170,6 +170,33 @@ int main(int argc, char** argv) CHECK(!parse(&res, sUrl) && res.sError, "over long scheme refused"); if(res.sError){ free(res.sError); res.sError = NULL; } + printf("INFO: Header lookup on a saved response\n"); + + /* The lookup only reads sHeaders, so a response can be staged by hand. + Mixed case and odd spacing on purpose, both occur in the wild. */ + memset(&res, 0, sizeof(DasHttpResp)); + res.sHeaders = strdup( + "HTTP/1.1 200 OK\r\n" + "Server: Apache\r\n" + "etag: \"5a6-607e10c29b347\"\r\n" + "Last-Modified: Tue, 17 Oct 2023 03:23:33 GMT \r\n" + "Content-Type: application/json\r\n" + "\r\n" + ); + CHECK(DasHttpResp_header(&res, "ETag", sOut, sizeof(sOut)) && streq(sOut, "\"5a6-607e10c29b347\""), + "ETag found without regard to case, got '%s'", sOut); + CHECK(DasHttpResp_header(&res, "last-modified", sOut, sizeof(sOut)) && + streq(sOut, "Tue, 17 Oct 2023 03:23:33 GMT"), "Last-Modified found and trimmed, got '%s'", sOut); + CHECK(DasHttpResp_header(&res, "Content-Type", sOut, sizeof(sOut)) && streq(sOut, "application/json"), + "Content-Type found"); + CHECK(!DasHttpResp_header(&res, "Content-Length", sOut, sizeof(sOut)) && (sOut[0] == '\0'), + "absent header reports false with an empty buffer"); + CHECK(!DasHttpResp_header(&res, "Content", sOut, sizeof(sOut)), "a prefix of a name is not a match"); + CHECK(!DasHttpResp_header(&res, "ETag", sOut, 8), "a value that does not fit is refused, not cut"); + free(res.sHeaders); + res.sHeaders = NULL; + CHECK(!DasHttpResp_header(&res, "ETag", sOut, sizeof(sOut)), "no headers, no match"); + printf("INFO: %d checks passed, %d failed\n", g_nPass, g_nFail); return (g_nFail == 0) ? 0 : 13; } diff --git a/test/das_cache_srv.py b/test/das_cache_srv.py new file mode 100755 index 0000000..0bb5027 --- /dev/null +++ b/test/das_cache_srv.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Throwaway HTTP file server for test/das_cache_test.sh. + +Serves the files in one directory the way a well behaved static server +does: ETag and Last-Modified on every 200, and 304 Not Modified when the +request's If-None-Match matches or its If-Modified-Since is not older than +the file. With --no-validators it behaves like a server that knows nothing +of caching, which is what dasflex does today. Every request is appended +to the log file as one line, so the shell test can see what the client +sent and what it got back. + +Drafted by Claude Fable 5.1, filed by Chris Piker. Python 3.6 compatible. +""" + +import argparse +import email.utils +import os +import sys +from http.server import BaseHTTPRequestHandler, HTTPServer + +g_dir = "." +g_log = None +g_validators = True + + +class Handler(BaseHTTPRequestHandler): + + def log_message(self, fmt, *args): + pass # the request log below is the one the test reads + + def _log(self, code, inm, ims): + with open(g_log, "a") as f: + f.write("GET %s %d inm=%s ims=%s\n" % ( + self.path, code, inm if inm else "-", ims if ims else "-" + )) + + def do_GET(self): + sPath = os.path.join(g_dir, self.path.lstrip("/").split("?")[0]) + inm = self.headers.get("If-None-Match") + ims = self.headers.get("If-Modified-Since") + + if not os.path.isfile(sPath): + self.send_response(404) + self.end_headers() + self._log(404, inm, ims) + return + + st = os.stat(sPath) + nMod = int(st.st_mtime) + sETag = '"%x-%x"' % (int(st.st_mtime * 1000), st.st_size) + sLastMod = email.utils.formatdate(nMod, usegmt=True) + + if g_validators: + bCurrent = False + if inm is not None: + bCurrent = (inm == sETag) + elif ims is not None: + try: + bCurrent = email.utils.parsedate_to_datetime(ims).timestamp() >= nMod + except (TypeError, ValueError): + bCurrent = False + if bCurrent: + self.send_response(304) + self.send_header("ETag", sETag) + self.send_header("Last-Modified", sLastMod) + self.end_headers() + self._log(304, inm, ims) + return + + with open(sPath, "rb") as f: + xBody = f.read() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(xBody))) + if g_validators: + self.send_header("ETag", sETag) + self.send_header("Last-Modified", sLastMod) + self.end_headers() + self.wfile.write(xBody) + self._log(200, inm, ims) + + +def main(): + global g_dir, g_log, g_validators + psr = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + psr.add_argument("--dir", required=True, help="directory to serve") + psr.add_argument("--log", required=True, help="request log, appended") + psr.add_argument("--port-file", required=True, help="the chosen port is written here") + psr.add_argument("--no-validators", action="store_true", help="send no ETag or Last-Modified") + opts = psr.parse_args() + + g_dir = opts.dir + g_log = opts.log + g_validators = not opts.no_validators + + srv = HTTPServer(("127.0.0.1", 0), Handler) + with open(opts.port_file, "w") as f: + f.write("%d\n" % srv.server_port) + try: + srv.serve_forever() + except KeyboardInterrupt: + pass + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/das_cache_test.sh b/test/das_cache_test.sh new file mode 100755 index 0000000..38ed798 --- /dev/null +++ b/test/das_cache_test.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash + +# Catalog node cache: the second run makes no request, an aged copy is +# revalidated with If-None-Match and renewed by a 304, a changed node is +# fetched again, an unreachable server leaves the old copy in use, and a +# server without validators still works, just without the saving. +# +# Runs against test/das_cache_srv.py on a loopback port picked at random, +# so nothing leaves the machine. The fixtures are copied to a scratch +# directory so one of them can be touched. Skips, passing, when python3 +# is not available. +# +# Drafted by Claude Fable 5.1, filed by Chris Piker. + +BD=$1 +FETCH="$BD/das_fetch" +FX=test/das_fetch + +if ! command -v python3 > /dev/null 2>&1; then + echo " SKIP: python3 not found, cache revalidation test not run" + exit 0 +fi + +nFail=0 +nPass=0 +check() { # check DESCRIPTION CONDITION... + local sWhat=$1; shift + if "$@"; then nPass=$((nPass + 1)); else echo " FAIL: $sWhat"; nFail=$((nFail + 1)); fi +} + +TMPD=$(mktemp -d) +SRVPID="" +cleanup() { [ -n "$SRVPID" ] && kill "$SRVPID" 2>/dev/null; rm -rf "$TMPD"; } +trap cleanup EXIT + +SRV=$TMPD/srv; mkdir -p "$SRV"; cp $FX/geomag_flex_0.7.json "$SRV/" +LOG=$TMPD/requests.log; : > "$LOG" +export DAS_USERDIR=$TMPD/userdir +CACHE=$DAS_USERDIR/cache/nodes + +start_server() { # start_server [--no-validators] + rm -f "$TMPD/port" + python3 test/das_cache_srv.py --dir "$SRV" --log "$LOG" --port-file "$TMPD/port" "$@" & + SRVPID=$! + for i in 1 2 3 4 5 6 7 8 9 10; do [ -s "$TMPD/port" ] && break; sleep 0.5; done + [ -s "$TMPD/port" ] || { echo " FAIL: test server did not start"; exit 1; } + PORT=$(cat "$TMPD/port") +} +stop_server() { kill "$SRVPID" 2>/dev/null; wait "$SRVPID" 2>/dev/null; SRVPID=""; } + +mtime() { python3 -c "import os,sys; print(int(os.stat(sys.argv[1]).st_mtime))" "$1"; } + +EXPECT="http://127.0.0.1:8765/data/geomag.d2s?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" + +start_server +U="http://127.0.0.1:$PORT/geomag_flex_0.7.json" + +# 1. First contact: a plain GET, a 200, a cached body and its validators +sOut=$($FETCH url $U 0 2>/dev/null) +check "first run resolves the node" [ "$sOut" == "$EXPECT" ] +check "first run made exactly one request" [ "$(wc -l < "$LOG")" -eq 1 ] +check "first request was unconditional" grep -q "200 inm=- ims=-" "$LOG" +BODY=$(ls "$CACHE"/*.json 2>/dev/null | head -1) +check "a cached copy was written" [ -n "$BODY" ] && [ -s "$BODY" ] +check "its validators were saved beside it" grep -q "^ETag: " "$BODY.hdr" +check "Last-Modified saved too" grep -q "^Last-Modified: " "$BODY.hdr" + +# 2. Inside the trust window nothing is sent at all +sOut=$($FETCH url $U 0 2>/dev/null) +check "second run resolves from the cache" [ "$sOut" == "$EXPECT" ] +check "second run made no request" [ "$(wc -l < "$LOG")" -eq 1 ] + +# 3. -f forces a check: conditional GET, 304, body not resent, copy renewed +sleep 1.1 +nBefore=$(mtime "$BODY") +sOut=$($FETCH -f url $U 0 2>/dev/null) +check "forced check still resolves" [ "$sOut" == "$EXPECT" ] +check "forced check made one request" [ "$(wc -l < "$LOG")" -eq 2 ] +sETag=$(sed -n 's/^ETag: //p' "$BODY.hdr") +check "it carried the saved ETag" grep -q "304 inm=$sETag ims=" "$LOG" +check "it carried the saved date too" grep -qv "ims=-" <(tail -1 "$LOG") +check "the copy's age was renewed" [ "$(mtime "$BODY")" -gt "$nBefore" ] + +# 4. The node changes on the server: the next check downloads it again +sleep 1.1 +touch "$SRV/geomag_flex_0.7.json" +sOut=$($FETCH -f url $U 0 2>/dev/null) +check "changed node still resolves" [ "$sOut" == "$EXPECT" ] +check "changed node came back as a 200" grep -q "^GET .* 200 inm=$sETag" <(tail -1 "$LOG") +sNewETag=$(sed -n 's/^ETag: //p' "$BODY.hdr") +check "new validators replaced the old" [ -n "$sNewETag" ] && [ "$sNewETag" != "$sETag" ] + +# 5. Server gone: a forced check fails over to the cached copy with a warning +stop_server +sErr=$($FETCH -f url $U 0 2>&1 >/dev/null); nRet=$? +sOut=$($FETCH -f url $U 0 2>/dev/null) +check "unreachable server, cached copy used" [ "$sOut" == "$EXPECT" ] +check "and the run succeeds" [ $nRet -eq 0 ] +check "with a warning that says so" grep -q "cached copy" <<< "$sErr" + +# 6. A server with no validators: every forced check is a full fetch, no side file +rm -rf "$DAS_USERDIR"; : > "$LOG" +start_server --no-validators +U="http://127.0.0.1:$PORT/geomag_flex_0.7.json" +sOut=$($FETCH url $U 0 2>/dev/null) +check "plain server resolves" [ "$sOut" == "$EXPECT" ] +BODY=$(ls "$CACHE"/*.json 2>/dev/null | head -1) +check "plain server copy cached" [ -n "$BODY" ] +check "but no validators to save" [ ! -e "$BODY.hdr" ] +sOut=$($FETCH -f url $U 0 2>/dev/null) +check "forced check is unconditional" grep -q "200 inm=- ims=-" <(tail -1 "$LOG") +check "two full fetches in all" [ "$(grep -c " 200 " "$LOG")" -eq 2 ] +stop_server + +echo +if [ $nFail -eq 0 ]; then + echo " Result: PASSED ($nPass checks)" + exit 0 +else + echo " Result: FAILED ($nFail of $((nPass + nFail)) checks)" + exit 1 +fi From dfe5814c00932992155ec9c6caf5f9ccd1afc7da Mon Sep 17 00:00:00 2001 From: cpiker Date: Mon, 26 Oct 2026 10:28:40 -0500 Subject: [PATCH 07/18] settable: the adjustable properties of a data source DasSettable describes one changeable interface property of a source node: its key, type, units, default, allowed values and the protocol parameter that carries it. TestSettable adds in-tree validation tests Developed-with: Claude Fable 5.1 --- buildfiles/Darwin.mak | 2 + buildfiles/Linux.mak | 2 + buildfiles/Windows.mak | 2 + buildfiles/cmake/das2CTests.cmake | 1 + buildfiles/sources.mak | 6 +- das3/core.h | 1 + das3/defs.h | 3 +- das3/sett_priv.h | 145 ++++++++ das3/settable.c | 537 ++++++++++++++++++++++++++++++ das3/settable.h | 233 +++++++++++++ das3/util.h | 2 + test/TestSettable.c | 415 +++++++++++++++++++++++ 12 files changed, 1345 insertions(+), 4 deletions(-) create mode 100644 das3/sett_priv.h create mode 100644 das3/settable.c create mode 100644 das3/settable.h create mode 100644 test/TestSettable.c diff --git a/buildfiles/Darwin.mak b/buildfiles/Darwin.mak index c59b91f..64ea503 100644 --- a/buildfiles/Darwin.mak +++ b/buildfiles/Darwin.mak @@ -376,6 +376,8 @@ test_main: $(BD) $(BD)/$(TARG).a $(BUILD_TEST_PROGS) $(BULID_UTIL_PROGS) @$(BD)/TestCredMngr $(BD) @echo "INFO: Running unit test for catalog sessions over local nodes, $(BD)/TestCat..." @$(BD)/TestCat + @echo "INFO: Running unit test for source settings over local nodes, $(BD)/TestSettable..." + @$(BD)/TestSettable @echo "INFO: Running unit test for the URL parser, $(BD)/TestHttp..." @$(BD)/TestHttp @echo "INFO: Running unit test for exception names and send.h writers, $(BD)/TestOob..." diff --git a/buildfiles/Linux.mak b/buildfiles/Linux.mak index a0b0893..0880c0a 100644 --- a/buildfiles/Linux.mak +++ b/buildfiles/Linux.mak @@ -374,6 +374,8 @@ test_main: $(BD) $(BD)/$(TARG).a $(BUILD_TEST_PROGS) $(BUILD_UTIL_PROGS) @$(BD)/TestCredMngr $(BD) @echo "INFO: Running unit test for catalog sessions over local nodes, $(BD)/TestCat..." @$(BD)/TestCat + @echo "INFO: Running unit test for source settings over local nodes, $(BD)/TestSettable..." + @$(BD)/TestSettable @echo "INFO: Running unit test for the URL parser, $(BD)/TestHttp..." @$(BD)/TestHttp @echo "INFO: Running unit test for exception names and send.h writers, $(BD)/TestOob..." diff --git a/buildfiles/Windows.mak b/buildfiles/Windows.mak index eb7d98f..8c61f00 100644 --- a/buildfiles/Windows.mak +++ b/buildfiles/Windows.mak @@ -390,6 +390,8 @@ test_main: @$(BD)\TestCredMngr.exe $(BD) @echo INFO: Running unit test for catalog sessions over local nodes, $(BD)\TestCat... @$(BD)\TestCat.exe + @echo INFO: Running unit test for source settings over local nodes, $(BD)\TestSettable... + @$(BD)\TestSettable.exe @echo INFO: Running unit test for the URL parser, $(BD)\TestHttp... @$(BD)\TestHttp.exe @echo INFO: Running unit test for exception names and send.h writers, $(BD)\TestOob... diff --git a/buildfiles/cmake/das2CTests.cmake b/buildfiles/cmake/das2CTests.cmake index 717c917..b73b082 100644 --- a/buildfiles/cmake/das2CTests.cmake +++ b/buildfiles/cmake/das2CTests.cmake @@ -65,6 +65,7 @@ _das_test(TestFilter main TestFilter) _das_test(TestUri main TestUri "${_bd}/uri_tplt") _das_test(TestCredMngr main TestCredMngr "${_bd}") _das_test(TestCat main TestCat) +_das_test(TestSettable main TestSettable) _das_test(TestHttp main TestHttp) _das_test(TestOob main TestOob) foreach(_v 2 3) diff --git a/buildfiles/sources.mak b/buildfiles/sources.mak index 6f793aa..b8b2647 100644 --- a/buildfiles/sources.mak +++ b/buildfiles/sources.mak @@ -15,7 +15,7 @@ LIB_SRCS = das3/das1.c das3/array.c das3/buffer.c das3/builder.c das3/cli.c \ das3/dataset_hdr2.c das3/dataset_hdr3.c das3/datum.c das3/descriptor.c \ das3/dft.c das3/dimension.c das3/dsdf.c das3/encoding.c das3/http.c \ das3/io.c das3/iterator.c das3/json.c das3/toml.c das3/log.c das3/node.c \ - das3/catalog.c \ + das3/catalog.c das3/settable.c \ das3/oob.c das3/operator.c das3/packet.c das3/plane.c das3/processor.c \ das3/property.c das3/send.c das3/stream.c das3/time.c das3/tt2000.c \ das3/units.c das3/utf8.c das3/util.c das3/value.c \ @@ -30,7 +30,7 @@ LIB_HDRS = das3/defs.h das3/time.h das3/das1.h das3/util.h das3/log.h \ das3/dimension.h das3/dataset.h das3/plane.h das3/packet.h das3/stream.h \ das3/processor.h das3/property.h das3/oob.h das3/io.h das3/iterator.h \ das3/builder.h das3/dsdf.h das3/credentials.h das3/http.h das3/dft.h \ - das3/json.h das3/toml.h das3/node.h das3/catalog.h das3/cli.h das3/send.h das3/uri.h \ + das3/json.h das3/toml.h das3/node.h das3/catalog.h das3/settable.h das3/cli.h das3/send.h das3/uri.h \ das3/codec.h das3/codex.h das3/core.h das3/variable.h \ das3/form.h das3/form_rot.h das3/form_lin.h das3/form_pt.h \ das3/form_cplx.h das3/form_vec.h das3/form_pos.h das3/das2_compat.h \ @@ -61,7 +61,7 @@ CDF_UTIL_SRCS = utilities/das3_cdf.c utilities/das3_from_cdf.c \ utilities/das2_from_cdf.c TEST_SRCS = test/TestUnits.c test/TestArray.c test/TestDs.c test/TestBuilder.c \ - test/TestAuth.c test/TestCatalog.c test/TestCat.c test/TestHttp.c test/TestTT2000.c test/ex_das_cli.c \ + test/TestAuth.c test/TestCatalog.c test/TestCat.c test/TestSettable.c test/TestHttp.c test/TestTT2000.c test/ex_das_cli.c \ test/ex_das_ephem.c test/TestCredMngr.c test/TestV3Read.c test/TestFeed.c \ test/TestProp.c \ test/TestIter.c test/TestUri.c test/TestFilter.c test/TestValue.c \ diff --git a/das3/core.h b/das3/core.h index 1c91aa6..71ca0c0 100644 --- a/das3/core.h +++ b/das3/core.h @@ -233,6 +233,7 @@ das3/utf8.h Counting characters instead of bytes das3/toml.h Reading TOML configuration files das3/dft.h Properly scaled power spectral densities das3/catalog.h Finding data sources in the federated catalog system +das3/settable.h The adjustable properties of a data source das3/http.h Reading streams from servers, credentials.h handles logins das3/builder.h Collect a whole das2 or das3 stream into DasDs objects das3/packet.h das2 packet descriptors, new code should use DasDs instead diff --git a/das3/defs.h b/das3/defs.h index b1db56c..bfc5a69 100644 --- a/das3/defs.h +++ b/das3/defs.h @@ -270,7 +270,8 @@ typedef int DasErrCode; #define DASERR_FORM 43 #define DASERR_GEN 44 #define DASERR_CAT 45 -#define DASERR_MAX 45 +#define DASERR_SRC 46 +#define DASERR_MAX 46 #ifdef __cplusplus } diff --git a/das3/sett_priv.h b/das3/sett_priv.h new file mode 100644 index 0000000..861e924 --- /dev/null +++ b/das3/sett_priv.h @@ -0,0 +1,145 @@ +/* Copyright (C) 2026 Chris Piker + * + * This file is part of das2C, the Core Das2 C Library. + * + * Das2C is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License version 2.1 as published + * by the Free Software Foundation. + * + * Das2C is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for + * more details. + * + * You should have received a copy of the GNU Lesser General Public License + * version 2.1 along with das2C; if not, see . + */ + +/* Settable internals shared by settable.c and source.c. Not installed. + * + * The struct is here rather than in settable.c because source.c builds the + * list when a source node loads and walks it when matching keys, and C has + * no package-private. Applications see only the accessors in settable.h. + */ + +#ifndef _das_sett_priv_h_ +#define _das_sett_priv_h_ + +#include "settable.h" +#include "json.h" + +/* One settable. Borrowed pointers point into the node's parsed document, + * which the node keeps for its whole life, so nothing here is copied that + * the document already holds in a usable form. JSON number text is not + * NUL terminated, hence the owned copies of default, enum and range + * strings. + * + * Roughly a dozen pointers per settable. The hsk node, the largest in the + * wild, has 73 variables and comes to about 150 settables, so a list is a + * few tens of kilobytes rather than the 700 KB the fixed arrays of the + * first fetcher cost. + */ +struct das_settable { + das_src_sec_e sec; + das_settable_type_e type; /* resolved at parse, see _DasSettable_type() */ + + const char* sGroup; /* borrowed dict key, "" for options */ + const char* sProp; /* borrowed dict key, the node's spelling */ + char* sKey; /* owned, "coords.time.min" */ + + const DasJdo* pGroup; /* the coordinate, variable or format dict, + NULL for options */ + const DasJdo* pProp; /* the property dict: value, set, type, label */ + const DasJdo* pSet; /* the set dict: param, flag, pval, enum, range */ + const DasJdo* pParam; /* the protocol parameter entry for sParam, or + NULL if the node does not describe it */ + const char* sParam; /* borrowed from pSet, never NULL, see below */ + const char* sTypeName; /* borrowed, "" when inferred */ + bool bRequired; /* set/required or the parameter's required */ + + char* sDefVal; /* owned, NULL when the node's value is null */ + char** aEnum; /* owned, enum values in node order */ + size_t uEnum; + char* sLo; /* owned, range ends, NULL when none stated */ + char* sHi; +}; + +/* Build the settable list for a source node. + * + * Walks the four interface sections in das_src_sec_e order and makes one + * entry for every property that has a "set" dict, so entries come out + * grouped by section and then in document order, which is the order a + * settings panel should show them in. Both interface layouts in the wild + * are read: the 2018 one with "coordinates", properties beside the group's + * own label and units, and long property names; and the current one with + * "coords" and properties under "props". The layout is not reported, the + * entries look the same either way. + * + * A property whose set block has no "param" cannot be sent anywhere. Such + * an entry is dropped with a warning naming the key instead of failing the + * load, so a node with one bad entry still describes and serves the rest. + * Nothing else in the interface can fail: unknown types become text and + * missing titles are composed on demand. + * + * @param pIface The node's "interface" dict. NULL or a non-dict gives an + * empty list, which is not an error; a source may have nothing to + * set. + * + * @param pParams The protocol's parameter dict, "httpParams" or + * "http_params", used for the type, required flag and range of a + * settable that does not state its own. May be NULL. + * + * @param puCount Receives the number of entries. + * + * @returns A heap array of exactly *puCount entries, to be released with + * _DasSettable_freeList(), or NULL only when memory could not be + * had, in which case *puCount is 0 and an error was logged. An + * empty list is a non-NULL pointer to nothing in particular, so + * callers test the count, not the pointer. + */ +DasSettable* _DasSettable_parse( + const DasJdo* pIface, const DasJdo* pParams, size_t* puCount +); + +/* Release a list from _DasSettable_parse() and the strings it owns. The + * document the entries pointed into is not touched. */ +void _DasSettable_freeList(DasSettable* aList, size_t uCount); + +/* Key grammar helpers, see the "Keys" block in settable.h. Queries match + * user input with these; keeping them next to the parser means the + * spellings accepted on input and the spellings written into keys cannot + * drift apart. */ + +/* Fold a section name as a user may type it, "coord", "coords", + * "coordinate", "fmt", "opts" and so on, to a section. Case insensitive. + * Returns false if the word is not a section name. */ +bool _das_src_secByName(const char* sName, das_src_sec_e* pSec); + +/* The short spelling of a property name: "minimum" -> "min", "enable" -> + * "enabled". Names with no alias are returned as given. Used on both + * sides of a comparison so that either spelling matches either. */ +const char* _das_src_propName(const char* sProp); + +/* True if two property names are the same after _das_src_propName() and + * case folding. */ +bool _das_src_propEq(const char* sA, const char* sB); + +/* Split a dotted key into at most three parts. + * + * @param sKey The key as typed. + * @param aParts Receives the parts, each NUL terminated. DAS_MAX_ID_BUFSZ + * per part is enough for anything a node may contain. + * @returns The number of parts, 1 to 3, or 0 if the key is empty, has an + * empty part, has more than three parts, or has a part that does + * not fit. + */ +int _das_src_splitKey(const char* sKey, char aParts[3][DAS_MAX_ID_BUFSZ]); + +/* Copy a JSON scalar as text: strings as they are, numbers in their source + * notation, true and false as those words. Returns false for null, + * containers and missing values, leaving sBuf untouched. This is the one + * reader of JSON number text in the catalog code, since das_json keeps + * numbers as unterminated text and a default of 86.4 should stay 86.4. */ +bool _das_json_scalar(const DasJdo* pJdo, char* sBuf, size_t uLen); + +#endif /* _das_sett_priv_h_ */ diff --git a/das3/settable.c b/das3/settable.c new file mode 100644 index 0000000..08e8698 --- /dev/null +++ b/das3/settable.c @@ -0,0 +1,537 @@ +/* Copyright (C) 2026 Chris Piker + * + * This file is part of das2C, the Core Das2 C Library. + * + * Das2C is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License version 2.1 as published + * by the Free Software Foundation. + * + * Das2C is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for + * more details. + * + * You should have received a copy of the GNU Lesser General Public License + * version 2.1 along with das2C; if not, see . + */ + +#define _POSIX_C_SOURCE 200112L + +#include +#include +#include +#include + +#include "util.h" +#include "log.h" +#include "time.h" +#include "sett_priv.h" + +/* Section names as the current catalogs spell them. Keys are built from + * these; input accepts the singular forms too, see _das_src_secByName(). */ +static const char* g_sSecName[das_src_sec_size] = {"coords", "data", "formats", "options"}; + +/* ************************************************************************* */ +/* Small string and JSON helpers. DasJdo_get() takes slash paths, so the + * dots in parameter names such as "read.time.min" are not special to it. */ + +static bool _strieq(const char* a, const char* b) +{ + if((a == NULL)||(b == NULL)) return false; + while(*a && *b){ + if(tolower((unsigned char)*a) != tolower((unsigned char)*b)) return false; + ++a; ++b; + } + return (*a == '\0') && (*b == '\0'); +} + +static void _strcpyz(char* sDest, const char* sSrc, size_t uDest) +{ + if(uDest == 0) return; + strncpy(sDest, sSrc ? sSrc : "", uDest - 1); + sDest[uDest - 1] = '\0'; +} + +static const DasJdo* _jget(const DasJdo* p, const char* sPath) +{ + return ((p == NULL)||(sPath == NULL)) ? NULL : DasJdo_get(p, sPath); +} + +static const char* _jstr(const DasJdo* p, const char* sPath) +{ + const DasJdo* pSub = _jget(p, sPath); + return ((pSub != NULL)&&(pSub->type == das_json_type_str)) ? DasJdo_string(pSub) : NULL; +} + +static bool _jisdict(const DasJdo* p){ return (p != NULL)&&(p->type == das_json_type_dict); } +static bool _jisary(const DasJdo* p){ return (p != NULL)&&(p->type == das_json_type_ary); } +static bool _jtrue(const DasJdo* p){ return (p != NULL)&&(p->type == das_json_type_true); } +static bool _jisbool(const DasJdo* p){ + return (p != NULL)&&((p->type == das_json_type_true)||(p->type == das_json_type_false)); +} + +bool _das_json_scalar(const DasJdo* p, char* sBuf, size_t uLen) +{ + if((p == NULL)||(uLen == 0)) return false; + switch(p->type){ + case das_json_type_str: + _strcpyz(sBuf, ((const das_json_str*)p->value)->string, uLen); + return true; + case das_json_type_num:{ + const das_json_num* pNum = (const das_json_num*)p->value; + size_t u = (pNum->number_size < uLen - 1) ? pNum->number_size : uLen - 1; + memcpy(sBuf, pNum->number, u); + sBuf[u] = '\0'; + return true; + } + case das_json_type_true: _strcpyz(sBuf, "true", uLen); return true; + case das_json_type_false: _strcpyz(sBuf, "false", uLen); return true; + default: return false; + } +} + +/* A heap copy of a scalar, NULL for anything that is not one. Defaults, + * enum entries and range ends are short in practice; the longest seen is a + * few dozen bytes, so anything past this is cut rather than carried. */ +#define SCALAR_SZ 1024 +static char* _scalarDup(const DasJdo* p) +{ + char sBuf[SCALAR_SZ]; + if(!_das_json_scalar(p, sBuf, sizeof(sBuf))) return NULL; + return das_strdup(sBuf); +} + +static bool _parseBool(const char* s, bool* pOut) +{ + if(s == NULL) return false; + if(_strieq(s,"true")||_strieq(s,"yes")||_strieq(s,"on")||_strieq(s,"1")|| + _strieq(s,"t")||_strieq(s,"y")){ *pOut = true; return true; } + if(_strieq(s,"false")||_strieq(s,"no")||_strieq(s,"off")||_strieq(s,"0")|| + _strieq(s,"f")||_strieq(s,"n")){ *pOut = false; return true; } + return false; +} + +/* ************************************************************************* */ +/* Key grammar */ + +const char* das_src_sec_name(das_src_sec_e sec) +{ + return ((sec >= 0)&&(sec < das_src_sec_size)) ? g_sSecName[sec] : ""; +} + +bool _das_src_secByName(const char* s, das_src_sec_e* pSec) +{ + if(_strieq(s,"coord")||_strieq(s,"coords")||_strieq(s,"coordinate")||_strieq(s,"coordinates")){ + *pSec = das_src_sec_coord; return true; + } + if(_strieq(s,"data")){ *pSec = das_src_sec_data; return true; } + if(_strieq(s,"format")||_strieq(s,"formats")||_strieq(s,"fmt")){ + *pSec = das_src_sec_format; return true; + } + if(_strieq(s,"option")||_strieq(s,"options")||_strieq(s,"opt")||_strieq(s,"opts")){ + *pSec = das_src_sec_option; return true; + } + return false; +} + +const char* _das_src_propName(const char* s) +{ + if(s == NULL) return ""; + if(_strieq(s, "minimum")||_strieq(s, "min")) return "min"; + if(_strieq(s, "maximum")||_strieq(s, "max")) return "max"; + if(_strieq(s, "resolution")||_strieq(s, "res")) return "res"; + if(_strieq(s, "interval")||_strieq(s, "inter")) return "inter"; + if(_strieq(s, "enable")||_strieq(s, "enabled")) return "enabled"; + return s; +} + +bool _das_src_propEq(const char* a, const char* b) +{ + return _strieq(_das_src_propName(a), _das_src_propName(b)); +} + +int _das_src_splitKey(const char* sKey, char aParts[3][DAS_MAX_ID_BUFSZ]) +{ + if(sKey == NULL) return 0; + int n = 0; + const char* pBeg = sKey; + while(true){ + const char* pEnd = strchr(pBeg, '.'); + size_t uLen = pEnd ? (size_t)(pEnd - pBeg) : strlen(pBeg); + if((uLen == 0)||(uLen >= DAS_MAX_ID_BUFSZ)||(n == 3)) return 0; + memcpy(aParts[n], pBeg, uLen); + aParts[n][uLen] = '\0'; + ++n; + if(pEnd == NULL) break; + pBeg = pEnd + 1; + } + return n; +} + +/* ************************************************************************* */ +/* Parsing */ + +/* Fold a type word from a node to the enum. NULL and unknown words are + * text, the caller decides whether that is final. */ +static bool _typeByName(const char* s, das_settable_type_e* pType) +{ + if(s == NULL) return false; + if(_strieq(s,"boolean")||_strieq(s,"bool")){ *pType = das_settable_bool; return true; } + if(_strieq(s,"integer")||_strieq(s,"int")){ *pType = das_settable_int; return true; } + if(_strieq(s,"real")||_strieq(s,"double")||_strieq(s,"float")){ + *pType = das_settable_real; return true; + } + if(_strieq(s,"isotime")||_strieq(s,"time")||_strieq(s,"datetime")){ + *pType = das_settable_time; return true; + } + if(_strieq(s,"enum")){ *pType = das_settable_enum; return true; } + if(_strieq(s,"string")||_strieq(s,"text")){ *pType = das_settable_text; return true; } + return false; +} + +/* Preference order: a set enum list, the property's own type, a boolean + * default, a flag toggle, the protocol parameter's type, then text. The + * enum list goes first because a property declared "string" with a list of + * choices is still a list of choices to anyone presenting it. A flag set + * parameter type is a server side packing detail and says nothing about + * the single toggle that feeds it. */ +static void _resolveType(DasSettable* p) +{ + p->sTypeName = _jstr(p->pProp, "type"); + if(p->sTypeName == NULL) p->sTypeName = ""; + + if(p->uEnum > 0){ p->type = das_settable_enum; return; } + if(_typeByName(p->sTypeName, &(p->type))){ + if(p->type != das_settable_enum) return; + } + if(_jisbool(_jget(p->pProp, "value")) || (_jstr(p->pSet, "flag") != NULL)){ + p->type = das_settable_bool; return; + } + const char* sParamType = _jstr(p->pParam, "type"); + if(_typeByName(sParamType, &(p->type)) && (p->type != das_settable_enum)){ + if(p->sTypeName[0] == '\0') p->sTypeName = sParamType; + return; + } + p->type = das_settable_text; +} + +/* Enum entries are usually dicts carrying value plus per item pval or flag + * overrides, but a bare scalar list is accepted too. */ +static bool _fillEnum(DasSettable* p) +{ + const DasJdo* pEnum = _jget(p->pSet, "enum"); + if(!_jisary(pEnum)) return true; + + size_t uMax = ((const das_json_ary*)pEnum->value)->length; + if(uMax == 0) return true; + p->aEnum = (char**)calloc(uMax, sizeof(char*)); + if(p->aEnum == NULL) return false; + + for(const das_json_ary_el* pEl = DasJdo_aryFirst(pEnum); pEl; pEl = pEl->next){ + const DasJdo* pVal = _jisdict(pEl->value) ? _jget(pEl->value, "value") : pEl->value; + char* s = _scalarDup(pVal); + if(s == NULL) continue; + p->aEnum[p->uEnum] = s; + p->uEnum += 1; + } + return true; +} + +static void _fillRange(DasSettable* p) +{ + const DasJdo* pRange = _jget(p->pSet, "range"); + if(!_jisary(pRange)) pRange = _jget(p->pParam, "range"); + if(!_jisary(pRange) && (p->sec == das_src_sec_coord) && + (_das_src_propEq(p->sProp, "min") || _das_src_propEq(p->sProp, "max"))) + pRange = _jget(p->pGroup, "validRange"); + if(!_jisary(pRange)) return; + + const das_json_ary_el* pEl = DasJdo_aryFirst(pRange); + if(pEl == NULL) return; + p->sLo = _scalarDup(pEl->value); + if(pEl->next) p->sHi = _scalarDup(pEl->next->value); +} + +/* One property. With aOut NULL only the count is wanted. Returns false + * on allocation failure, the only way filling can fail. */ +static bool _addProp( + das_src_sec_e sec, const char* sGroup, const DasJdo* pGroup, + const char* sProp, const DasJdo* pProp, const DasJdo* pParams, + DasSettable* aOut, size_t* puCount +){ + if(!_jisdict(pProp)) return true; + const DasJdo* pSet = _jget(pProp, "set"); + if(!_jisdict(pSet)) return true; /* documentation only, not settable */ + + const char* sParam = _jstr(pSet, "param"); + if(sParam == NULL){ + if(aOut != NULL) + daslog_warn_v("Ignoring %s.%s%s%s, its set block names no parameter", + g_sSecName[sec], sGroup, sGroup[0] ? "." : "", sProp); + return true; + } + + if(aOut == NULL){ *puCount += 1; return true; } + + DasSettable* p = aOut + *puCount; + memset(p, 0, sizeof(DasSettable)); + p->sec = sec; + p->sGroup = sGroup; + p->sProp = sProp; + p->pGroup = pGroup; + p->pProp = pProp; + p->pSet = pSet; + p->sParam = sParam; + p->pParam = _jget(pParams, sParam); + p->bRequired = _jtrue(_jget(pSet, "required")) || _jtrue(_jget(p->pParam, "required")); + + size_t uKey = strlen(g_sSecName[sec]) + strlen(sGroup) + strlen(sProp) + 3; + p->sKey = (char*)malloc(uKey); + if(p->sKey == NULL) return false; + if(sGroup[0] == '\0') + snprintf(p->sKey, uKey, "%s.%s", g_sSecName[sec], sProp); + else + snprintf(p->sKey, uKey, "%s.%s.%s", g_sSecName[sec], sGroup, sProp); + + p->sDefVal = _scalarDup(_jget(pProp, "value")); + if(!_fillEnum(p)) return false; + _fillRange(p); + _resolveType(p); + + *puCount += 1; + return true; +} + +/* A group is a coordinate, a data variable, a format, or the whole options + * block. Current groups keep properties under "props", 2018 groups hold + * them directly beside label/title/units. Anything without a 'set' is + * skipped either way, so iterating the group itself is safe for the old + * layout. */ +static bool _addGroup( + das_src_sec_e sec, const char* sGroup, const DasJdo* pGroup, + const DasJdo* pParams, DasSettable* aOut, size_t* puCount +){ + if(!_jisdict(pGroup)) return true; + const DasJdo* pProps = _jget(pGroup, "props"); + const DasJdo* pIter = _jisdict(pProps) ? pProps : pGroup; + for(const das_json_dict_el* pEl = DasJdo_dictFirst(pIter); pEl; pEl = pEl->next){ + if(!_addProp(sec, sGroup, pGroup, pEl->name->string, pEl->value, pParams, aOut, puCount)) + return false; + } + return true; +} + +static bool _addSection( + das_src_sec_e sec, const DasJdo* pSec, const DasJdo* pParams, + DasSettable* aOut, size_t* puCount +){ + if(!_jisdict(pSec)) return true; + for(const das_json_dict_el* pEl = DasJdo_dictFirst(pSec); pEl; pEl = pEl->next){ + if(!_addGroup(sec, pEl->name->string, pEl->value, pParams, aOut, puCount)) + return false; + } + return true; +} + +DasSettable* _DasSettable_parse(const DasJdo* pIface, const DasJdo* pParams, size_t* puCount) +{ + *puCount = 0; + if(!_jisdict(pIface)) pIface = NULL; + + /* Two passes over the same walk: count, allocate exactly that many, + fill. Options are a group with no enclosing section, so they go + through _addGroup directly and their pGroup is NULL. */ + size_t uCount = 0; + const DasJdo* pSec = _jget(pIface, "coords"); + if(pSec == NULL) pSec = _jget(pIface, "coordinates"); + const DasJdo* pOpts = _jget(pIface, "options"); + + _addSection(das_src_sec_coord, pSec, pParams, NULL, &uCount); + _addSection(das_src_sec_data, _jget(pIface, "data"), pParams, NULL, &uCount); + _addSection(das_src_sec_format, _jget(pIface, "formats"), pParams, NULL, &uCount); + _addGroup(das_src_sec_option, "", pOpts, pParams, NULL, &uCount); + + /* An empty list is still a list: one byte keeps malloc from handing + back NULL on some platforms and lets callers test the count alone. */ + DasSettable* aOut = (DasSettable*)calloc(uCount ? uCount : 1, sizeof(DasSettable)); + if(aOut == NULL){ + daslog_error_v("Couldn't allocate %zu source settings", uCount); + return NULL; + } + + size_t uFill = 0; + bool bOk = + _addSection(das_src_sec_coord, pSec, pParams, aOut, &uFill) && + _addSection(das_src_sec_data, _jget(pIface, "data"), pParams, aOut, &uFill) && + _addSection(das_src_sec_format, _jget(pIface, "formats"), pParams, aOut, &uFill) && + _addGroup(das_src_sec_option, "", pOpts, pParams, aOut, &uFill); + + if(!bOk){ + daslog_error_v("Couldn't allocate the strings of source setting %zu", uFill); + _DasSettable_freeList(aOut, uFill + 1); + return NULL; + } + *puCount = uFill; + return aOut; +} + +void _DasSettable_freeList(DasSettable* aList, size_t uCount) +{ + if(aList == NULL) return; + for(size_t u = 0; u < uCount; ++u){ + DasSettable* p = aList + u; + free(p->sKey); + free(p->sDefVal); + for(size_t v = 0; v < p->uEnum; ++v) free(p->aEnum[v]); + free(p->aEnum); + free(p->sLo); + free(p->sHi); + } + free(aList); +} + +/* ************************************************************************* */ +/* Accessors */ + +das_src_sec_e DasSettable_section(const DasSettable* p){ return p->sec; } +const char* DasSettable_group(const DasSettable* p){ return p->sGroup; } +const char* DasSettable_prop(const DasSettable* p){ return p->sProp; } +const char* DasSettable_key(const DasSettable* p){ return p->sKey; } +const char* DasSettable_param(const DasSettable* p){ return p->sParam; } +bool DasSettable_required(const DasSettable* p){ return p->bRequired; } +const char* DasSettable_defVal(const DasSettable* p){ return p->sDefVal; } +das_settable_type_e DasSettable_type(const DasSettable* p){ return p->type; } +const char* DasSettable_typeName(const DasSettable* p){ return p->sTypeName; } +size_t DasSettable_enumCount(const DasSettable* p){ return p->uEnum; } + +const char* DasSettable_enum(const DasSettable* p, size_t uIdx) +{ + return (uIdx < p->uEnum) ? p->aEnum[uIdx] : NULL; +} + +bool DasSettable_range(const DasSettable* p, const char** psLo, const char** psHi) +{ + if((p->sLo == NULL)&&(p->sHi == NULL)) return false; + if(psLo) *psLo = p->sLo ? p->sLo : ""; + if(psHi) *psHi = p->sHi ? p->sHi : ""; + return true; +} + +/* Units for display. Current nodes put a units string on the property; + * both layouts may carry a units dict with a "value" at the group level, + * under props in the current layout and directly in the old one. */ +const char* DasSettable_units(const DasSettable* p) +{ + const DasJdo* pU = _jget(p->pProp, "units"); + if(pU != NULL){ + if(pU->type == das_json_type_str) return DasJdo_string(pU); + if(_jstr(pU, "value")) return _jstr(pU, "value"); + } + if(p->sec == das_src_sec_coord){ + const char* s = _jstr(p->pGroup, "props/units/value"); + if(s == NULL) s = _jstr(p->pGroup, "units/value"); + if(s != NULL) return s; + } + if(p->type == das_settable_time) return "UTC"; + return ""; +} + +/* What to call a group in composed text: its label, name or title, falling + * back to the dict key */ +static const char* _groupLabel(const DasSettable* p) +{ + const char* s = _jstr(p->pGroup, "label"); + if(s == NULL) s = _jstr(p->pGroup, "name"); + if(s == NULL) s = _jstr(p->pGroup, "title"); + return s ? s : p->sGroup; +} + +const char* DasSettable_title(const DasSettable* p, char* sBuf, size_t uLen) +{ + const char* s = _jstr(p->pProp, "title"); + if(s == NULL) s = _jstr(p->pProp, "summary"); + if(s == NULL) s = _jstr(p->pProp, "label"); + if(s == NULL) s = _jstr(p->pProp, "name"); + if(s != NULL) return s; + if(p->sec == das_src_sec_option) return ""; + + const char* sProp = _das_src_propName(p->sProp); + if(strcmp(sProp, "enabled") == 0){ + if(p->sec == das_src_sec_format){ + snprintf(sBuf, uLen, "Output as %s", _groupLabel(p)); + } + else{ + /* A variable's title is usually the physical quantity, which + reads better after "Output" than its short label does */ + const char* sTitle = _jstr(p->pGroup, "title"); + snprintf(sBuf, uLen, "Output %s", sTitle ? sTitle : _groupLabel(p)); + } + return sBuf; + } + if(p->sec == das_src_sec_coord){ + const char* sFmt = NULL; + if(strcmp(sProp, "min") == 0) sFmt = "Minimum %s value to output"; + else if(strcmp(sProp, "max") == 0) sFmt = "Maximum %s value to output"; + else if(strcmp(sProp, "res") == 0) sFmt = "Average over %s bins of this width"; + else if(strcmp(sProp, "inter") == 0) sFmt = "Step between output %s values"; + if(sFmt != NULL){ + snprintf(sBuf, uLen, sFmt, _groupLabel(p)); + return sBuf; + } + } + snprintf(sBuf, uLen, "%s %s", _groupLabel(p), p->sProp); + return sBuf; +} + +/* ************************************************************************* */ +/* Validation. Failures are logged and returned, not raised through + * das_error(), because these are user typed values and the default error + * disposition would exit the application over a typo. */ + +DasErrCode DasSettable_check(const DasSettable* p, const char* sVal) +{ + if(sVal == NULL) sVal = ""; + char* pEnd = NULL; + + switch(p->type){ + case das_settable_bool:{ + bool b; + if(_parseBool(sVal, &b)) return DAS_OKAY; + daslog_error_v("'%s' is not a boolean value for %s, use true or false", sVal, p->sKey); + return DASERR_SRC; + } + case das_settable_enum:{ + for(size_t u = 0; u < p->uEnum; ++u) + if(_strieq(p->aEnum[u], sVal)) return DAS_OKAY; + + char sChoices[512] = {'\0'}; + size_t u = 0; + for(size_t v = 0; (v < p->uEnum) && (u < sizeof(sChoices)); ++v) + u += snprintf(sChoices + u, sizeof(sChoices) - u, "%s%s", v ? ", " : "", p->aEnum[v]); + daslog_error_v("'%s' is not an allowed value for %s, choose one of: %s", + sVal, p->sKey, sChoices); + return DASERR_SRC; + } + case das_settable_int: + strtol(sVal, &pEnd, 10); + if((pEnd != sVal) && (*pEnd == '\0')) return DAS_OKAY; + daslog_error_v("'%s' is not an integer value for %s", sVal, p->sKey); + return DASERR_SRC; + case das_settable_real: + strtod(sVal, &pEnd); + if((pEnd != sVal) && (*pEnd == '\0')) return DAS_OKAY; + daslog_error_v("'%s' is not a real value for %s", sVal, p->sKey); + return DASERR_SRC; + case das_settable_time:{ + /* "now" is how nodes spell an open ended range and servers take + it in requests too */ + das_time dt; + if(_strieq(sVal, "now") || dt_parsetime(&dt, sVal)) return DAS_OKAY; + daslog_error_v("'%s' is not a time value for %s, use ISO 8601 such as " + "2026-01-15T12:00", sVal, p->sKey); + return DASERR_SRC; + } + default: + return DAS_OKAY; + } +} diff --git a/das3/settable.h b/das3/settable.h new file mode 100644 index 0000000..5951dab --- /dev/null +++ b/das3/settable.h @@ -0,0 +1,233 @@ +/* Copyright (C) 2026 Chris Piker + * + * This file is part of das2C, the Core Das2 C Library. + * + * Das2C is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License version 2.1 as published + * by the Free Software Foundation. + * + * Das2C is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for + * more details. + * + * You should have received a copy of the GNU Lesser General Public License + * version 2.1 along with das2C; if not, see . + */ + +/** @file settable.h The adjustable properties of a data source */ + +#ifndef _das_settable_h_ +#define _das_settable_h_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** @addtogroup catalog + * @{ + */ + +/** @defgroup settable Source settings + * + * A data source node describes what a client may ask for in its "interface" + * section: coordinate ranges, which data variables to include, output + * formats and server specific options. Each entry that a client may change + * carries a "set" block saying how a value for it reaches the server. + * The library flattens those entries into a list of DasSettable objects + * when a source node loads, one per changeable property, and the node + * hands them out by index. Everything else in the interface is + * documentation and is reached through DasNode_getJdo() as before. + * + * A DasSettable is a description. It says what a property is, what values + * it takes and what it is called, and it never changes after the node + * loads. Values chosen for a particular request live in a query object + * (see source.h), so one node can serve any number of queries at once. + * + * @anchor settable_keys + * ### Keys + * + * Every settable has a dotted key built from its section, group and + * property names as the node spells them, for example `coords.time.min`, + * `data.lshell.enabled`, `formats.csv.delim` or `options.text`. Options + * have no group, so their keys have two parts. Keys are how a request + * names a settable, and a query accepts the following shorter spellings as + * well, so long as the result is unambiguous within the source: + * + * - Section names in singular or plural form, `coord` or `coords`, + * `format` or `formats`, `option` or `options`, and `fmt` or `opt`. + * - The group and property alone, `time.min`, when only one section has + * that group. + * - A group alone, `lshell` or `data.lshell`, meaning the group's + * `enabled` toggle. + * - An option alone, `text`, when no group has that name. + * - The property names `min`, `max`, `res` and `inter` may be written + * out as `minimum`, `maximum`, `resolution` and `interval`, and + * `enable` as `enabled`. Nodes from 2018 spell them the long way, + * later nodes the short way, and users may type either. + * + * Matching is case insensitive. A spelling that fits more than one + * settable is an error that names the candidates, not a guess. Section + * names are ordered as in das_src_sec_e, which is also the order nodes + * present them in. + */ + +/** The parts of a source interface, in presentation order. + * + * `das_src_sec_size` is the number of real sections and is handy for + * sizing per section arrays. + */ +typedef enum das_src_sec_enum { + das_src_sec_coord = 0, /**< "coords": time, frequency, L-shell ... */ + das_src_sec_data, /**< "data": the variables a source can emit */ + das_src_sec_format, /**< "formats": output serializations and their knobs */ + das_src_sec_option, /**< "options": server specific switches, no groups */ + das_src_sec_size +} das_src_sec_e; + +/** The name of a section as it appears in keys, "coords", "data", + * "formats" or "options". Returns "" for das_src_sec_size. */ +DAS_API const char* das_src_sec_name(das_src_sec_e sec); + +/** The kind of value a settable takes. + * + * Source nodes name types with a handful of spellings that have drifted + * over the years. They are folded into this set when the node loads so + * that clients need not know the history. A type the library does not + * recognize is reported as das_settable_text and the node's spelling of it + * is still available via DasSettable_typeName(). + */ +typedef enum das_settable_type_enum { + das_settable_text = 0, /**< any string, including unrecognized types */ + das_settable_bool, /**< true or false, with the spellings DasSettable_check() allows */ + das_settable_int, /**< an integer in decimal */ + das_settable_real, /**< a real number */ + das_settable_time, /**< a UTC time, ISO 8601 preferred */ + das_settable_enum /**< one of a fixed list, see DasSettable_enumCount() */ +} das_settable_type_e; + +/** One changeable property of a data source. + * + * Settables are owned by the source node that lists them and are valid for + * as long as that node is. Applications never create or free them; they + * ask the node for them by index. All accessors take a const pointer and + * may be called from several threads at once. + */ +typedef struct das_settable DasSettable; + +/** The section of the interface this settable came from */ +DAS_API das_src_sec_e DasSettable_section(const DasSettable* pThis); + +/** The group name as the node spells it. + * + * For a coordinate that is the coordinate name, `time`; for a variable or + * format its name, `lshell` or `csv`. Options have no group and return + * "" (never NULL), so the result is always safe to print. + */ +DAS_API const char* DasSettable_group(const DasSettable* pThis); + +/** The property name as the node spells it, `min` or `minimum`, + * `enabled`, `delim`. Use DasSettable_key() rather than rebuilding keys + * from this, since the key is what a query matches against. */ +DAS_API const char* DasSettable_prop(const DasSettable* pThis); + +/** The full dotted key, see @ref settable_keys. Unique within a source. */ +DAS_API const char* DasSettable_key(const DasSettable* pThis); + +/** A short human readable title for the property. + * + * The node's own title is returned when it has one. Coordinate range + * properties and enable toggles usually have none, since their meaning is + * conventional, so a title is composed for them in sBuf from the group's + * label, for example "Minimum Time value to output" or "Output as CSV". + * Anything else without a title gets the group label and property name. + * + * @param sBuf Space for a composed title. 64 bytes covers the composed + * forms, longer group labels are cut to fit. + * @param uLen The size of sBuf. + * @returns A pointer to the title, which is either inside the node's + * document or is sBuf. Never NULL, but may be "" for an option + * with no title at all. + */ +DAS_API const char* DasSettable_title(const DasSettable* pThis, char* sBuf, size_t uLen); + +/** The units a value is expressed in, as a string the node supplies, or + * "UTC" for time typed settables with no stated units, or "" when there + * are none. Units are for display and are not checked against values. */ +DAS_API const char* DasSettable_units(const DasSettable* pThis); + +/** The kind of value this settable takes */ +DAS_API das_settable_type_e DasSettable_type(const DasSettable* pThis); + +/** The node's own spelling of the type, "isotime", "real", "boolean" and + * so on, or "" when the node gave none and the type was inferred. For + * display, DasSettable_type() is the one to branch on. */ +DAS_API const char* DasSettable_typeName(const DasSettable* pThis); + +/** True when a query must supply a value, or the server will refuse the + * request. A required settable with a default is satisfied by the default, + * see DasSettable_defVal(). */ +DAS_API bool DasSettable_required(const DasSettable* pThis); + +/** The value in effect when a query sets nothing, as the node spells it, + * or NULL if the node has no default. Numbers are given in the node's + * own notation, "86.4", not reformatted. */ +DAS_API const char* DasSettable_defVal(const DasSettable* pThis); + +/** The name of the protocol parameter that carries this setting to the + * server, "start_time" or "read.time.min". Several settables may share a + * parameter, boolean toggles that pack into one flag set in particular. + * Mostly of interest when composing requests by hand. */ +DAS_API const char* DasSettable_param(const DasSettable* pThis); + +/** The number of allowed values for an enumerated settable, 0 for any + * other type. */ +DAS_API size_t DasSettable_enumCount(const DasSettable* pThis); + +/** Allowed value number uIdx of an enumerated settable, as the node spells + * it, or NULL when uIdx is out of range. DasSettable_check() accepts + * these in any letter case; the value a query sends is the node's. */ +DAS_API const char* DasSettable_enum(const DasSettable* pThis, size_t uIdx); + +/** The stated range of a numeric or time settable. + * + * Nodes put ranges in several places: on the set block, on the protocol + * parameter, or for coordinate `min` and `max` on the coordinate itself as + * its valid range. All three are consulted, in that order. The upper end + * of a time range is often the word "now". + * + * @param psLo Receives the low end, as the node spells it. May be NULL. + * @param psHi Receives the high end. May be NULL. + * @returns true if a range was stated, false otherwise, in which case the + * output pointers are not touched. + */ +DAS_API bool DasSettable_range( + const DasSettable* pThis, const char** psLo, const char** psHi +); + +/** Decide if a string is an acceptable value for this settable. + * + * Booleans accept true, false, yes, no, on, off, 1 and 0 in any letter + * case. Enumerations accept any listed value, case insensitively. + * Integers, reals and times are checked for form only, not against the + * stated range, because servers are the authority on what they hold and + * ranges in nodes are routinely out of date. Text is always acceptable. + * + * Queries call this before storing a value, so applications rarely need to, + * but a settings panel may want to validate as the user types. + * + * @returns DAS_OKAY, or DASERR_SRC after logging a message that names the + * key, the offending value and, for an enumeration, the choices. + */ +DAS_API DasErrCode DasSettable_check(const DasSettable* pThis, const char* sVal); + +/** @} */ +/** @} */ + +#ifdef __cplusplus +} +#endif + +#endif /* _das_settable_h_ */ diff --git a/das3/util.h b/das3/util.h index bf2efd3..aa4e1f7 100644 --- a/das3/util.h +++ b/das3/util.h @@ -183,6 +183,8 @@ DAS_API DasErrCode das_error_func_fixed( * - @b 42 : uri.c - DASERR_URI * - @b 43 : form*.c - DASERR_FORM * - @b 44 : generator.c - DASERR_GEN + * - @b 45 : catalog.c - DASERR_CAT + * - @b 46 : settable.c - DASERR_SRC * * Application programs are recommended to use values 64 and above to avoid * colliding with future das2 error codes. diff --git a/test/TestSettable.c b/test/TestSettable.c new file mode 100644 index 0000000..e14de81 --- /dev/null +++ b/test/TestSettable.c @@ -0,0 +1,415 @@ +/* Copyright (C) 2026 Chris Piker + * + * This file is part of das2C, the Core Das2 C Library. + * + * Das2C is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License version 2.1 as published + * by the Free Software Foundation. + * + * Das2C is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for + * more details. + * + * You should have received a copy of the GNU Lesser General Public License + * version 2.1 along with das2C; if not, see . + */ + +/* Source settings parsed straight from the three saved source nodes under + * test/das_fetch, no DasNode involved. Run from the repository top. + * + * The private parser is driven directly because that is the seam the + * source node load (step 6) will call; the public accessors are then + * checked against what the fixtures say. geomag is the current layout + * with examples, hsk is the current layout at scale (73 variables), and + * juno_survey is the 2018 layout with long property names and flag packed + * options. */ + +#define _POSIX_C_SOURCE 200112L + +#include +#include +#include +#include + +#include +#include +#include "../das3/sett_priv.h" + +#define GEOMAG "test/das_fetch/geomag_flex_0.7.json" +#define HSK "test/das_fetch/hsk_flex_0.7.json" +#define JUNO "test/das_fetch/juno_survey_0.6.json" + +static int g_nFail = 0; +static int g_nPass = 0; + +#define CHECK(cond, ...) do{ \ + if(cond){ ++g_nPass; } \ + else{ ++g_nFail; printf(" FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } \ +}while(0) + +static bool streq(const char* a, const char* b) +{ + if((a == NULL)||(b == NULL)) return (a == b); + return strcmp(a, b) == 0; +} + +/* Validation failures log at error level by design; keep the test output + readable while provoking them */ +static DasErrCode quietCheck(const DasSettable* p, const char* sVal) +{ + daslog_setlevel(DASLOG_CRIT); + DasErrCode n = DasSettable_check(p, sVal); + daslog_setlevel(DASLOG_WARN); + return n; +} + +static DasJdo* loadJson(const char* sFile) +{ + FILE* pIn = fopen(sFile, "rb"); + if(pIn == NULL){ printf(" Can't open %s, run from the repo top\n", sFile); return NULL; } + fseek(pIn, 0, SEEK_END); + long nLen = ftell(pIn); + fseek(pIn, 0, SEEK_SET); + char* sBuf = (char*)malloc(nLen + 1); + size_t uRead = fread(sBuf, 1, nLen, pIn); + fclose(pIn); + sBuf[uRead] = '\0'; + DasJdo* pDom = das_json_parse(sBuf, uRead); + free(sBuf); + return pDom; +} + +/* The settable list for a saved node, through the same two dicts the node + load will pass */ +static DasSettable* parseNode(const DasJdo* pDom, size_t* puCount) +{ + const DasJdo* pParams = DasJdo_get(pDom, "protocol/httpParams"); + if(pParams == NULL) pParams = DasJdo_get(pDom, "protocol/http_params"); + return _DasSettable_parse(DasJdo_get(pDom, "interface"), pParams, puCount); +} + +static const DasSettable* byKey(const DasSettable* aList, size_t uCount, const char* sKey) +{ + for(size_t u = 0; u < uCount; ++u) + if(streq(DasSettable_key(aList + u), sKey)) return aList + u; + return NULL; +} + +/* ************************************************************************* */ + +static void test_grammar(void) +{ + printf("Key grammar:\n"); + das_src_sec_e sec; + CHECK(_das_src_secByName("coord", &sec) && (sec == das_src_sec_coord), "coord"); + CHECK(_das_src_secByName("Coordinates", &sec) && (sec == das_src_sec_coord), "Coordinates"); + CHECK(_das_src_secByName("fmt", &sec) && (sec == das_src_sec_format), "fmt"); + CHECK(_das_src_secByName("opts", &sec) && (sec == das_src_sec_option), "opts"); + CHECK(!_das_src_secByName("time", &sec), "time is not a section"); + CHECK(!_das_src_secByName("", &sec), "empty is not a section"); + + CHECK(streq(das_src_sec_name(das_src_sec_coord), "coords"), "coords name"); + CHECK(streq(das_src_sec_name(das_src_sec_format), "formats"), "formats name"); + CHECK(streq(das_src_sec_name(das_src_sec_size), ""), "sentinel has no name"); + + CHECK(streq(_das_src_propName("Minimum"), "min"), "Minimum -> min"); + CHECK(streq(_das_src_propName("inter"), "inter"), "inter stays"); + CHECK(streq(_das_src_propName("enable"), "enabled"), "enable -> enabled"); + CHECK(streq(_das_src_propName("delim"), "delim"), "unknown passes through"); + CHECK(_das_src_propEq("maximum", "MAX"), "maximum == MAX"); + CHECK(!_das_src_propEq("res", "inter"), "res != inter"); + + char aParts[3][DAS_MAX_ID_BUFSZ]; + CHECK(_das_src_splitKey("coords.time.min", aParts) == 3, "three parts"); + CHECK(streq(aParts[1], "time") && streq(aParts[2], "min"), "parts content"); + CHECK(_das_src_splitKey("lshell", aParts) == 1, "one part"); + CHECK(_das_src_splitKey("a.b.c.d", aParts) == 0, "four parts is malformed"); + CHECK(_das_src_splitKey("a..c", aParts) == 0, "empty part is malformed"); + CHECK(_das_src_splitKey("", aParts) == 0, "empty key is malformed"); + CHECK(_das_src_splitKey(NULL, aParts) == 0, "NULL key is malformed"); +} + +static void test_scalar(void) +{ + printf("JSON scalars:\n"); + /* Two letter keys: DasJdo_get() does not resolve one character names */ + const char* sDoc = "{\"st\":\"x\",\"nm\":86.4,\"in\":42,\"tr\":true,\"fa\":false," + "\"nl\":null,\"ar\":[1],\"di\":{}}"; + DasJdo* pDom = das_json_parse(sDoc, strlen(sDoc)); + CHECK(pDom != NULL, "parse"); + char sBuf[32]; + CHECK(_das_json_scalar(DasJdo_get(pDom, "st"), sBuf, 32) && streq(sBuf, "x"), "string"); + CHECK(_das_json_scalar(DasJdo_get(pDom, "nm"), sBuf, 32) && streq(sBuf, "86.4"), "number text kept"); + CHECK(_das_json_scalar(DasJdo_get(pDom, "in"), sBuf, 32) && streq(sBuf, "42"), "integer"); + CHECK(_das_json_scalar(DasJdo_get(pDom, "tr"), sBuf, 32) && streq(sBuf, "true"), "true"); + CHECK(_das_json_scalar(DasJdo_get(pDom, "fa"), sBuf, 32) && streq(sBuf, "false"), "false"); + CHECK(!_das_json_scalar(DasJdo_get(pDom, "nl"), sBuf, 32), "null is not a scalar"); + CHECK(!_das_json_scalar(DasJdo_get(pDom, "ar"), sBuf, 32), "array is not a scalar"); + CHECK(!_das_json_scalar(DasJdo_get(pDom, "di"), sBuf, 32), "dict is not a scalar"); + CHECK(!_das_json_scalar(NULL, sBuf, 32), "NULL is not a scalar"); + CHECK(_das_json_scalar(DasJdo_get(pDom, "nm"), sBuf, 3) && streq(sBuf, "86"), "number cut to fit"); + free(pDom); +} + +static void test_geomag(void) +{ + printf("geomag, current layout:\n"); + DasJdo* pDom = loadJson(GEOMAG); + CHECK(pDom != NULL, "load"); + if(pDom == NULL) return; + + size_t uCount = 0; + DasSettable* aList = parseNode(pDom, &uCount); + CHECK(aList != NULL, "parse"); + + /* time min/max/inter, lshell enabled, then 13 format knobs */ + CHECK(uCount == 17, "17 settables, got %zu", uCount); + + /* Order is section then document order */ + CHECK((uCount > 4) && (DasSettable_section(aList) == das_src_sec_coord), "coords first"); + CHECK((uCount > 4) && (DasSettable_section(aList + 3) == das_src_sec_data), "data second"); + CHECK((uCount > 4) && (DasSettable_section(aList + 4) == das_src_sec_format), "formats third"); + + const DasSettable* p = byKey(aList, uCount, "coords.time.min"); + CHECK(p != NULL, "coords.time.min"); + if(p){ + CHECK(streq(DasSettable_group(p), "time"), "group"); + CHECK(streq(DasSettable_prop(p), "min"), "prop"); + CHECK(streq(DasSettable_param(p), "read.time.min"), "param via dotted name"); + CHECK(DasSettable_required(p), "required from the protocol parameter"); + CHECK(DasSettable_type(p) == das_settable_time, "type from the parameter"); + CHECK(streq(DasSettable_typeName(p), "isotime"), "type name from the parameter"); + CHECK(streq(DasSettable_units(p), "UTC"), "UTC for a time"); + CHECK(DasSettable_defVal(p) == NULL, "null value is no default"); + CHECK(quietCheck(p, "2026-01-01") == DAS_OKAY, "ISO date passes"); + CHECK(quietCheck(p, "2026-01-01T12:00:00") == DAS_OKAY, "ISO time passes"); + CHECK(quietCheck(p, "now") == DAS_OKAY, "now passes"); + CHECK(quietCheck(p, "yesterday-ish") == DASERR_SRC, "garbage fails"); + } + + p = byKey(aList, uCount, "coords.time.inter"); + CHECK(p != NULL, "coords.time.inter"); + if(p){ + CHECK(DasSettable_type(p) == das_settable_real, "real from the parameter"); + CHECK(streq(DasSettable_units(p), "s"), "units string on the property"); + CHECK(quietCheck(p, "60") == DAS_OKAY, "60 is real"); + CHECK(quietCheck(p, "6e1") == DAS_OKAY, "6e1 is real"); + CHECK(quietCheck(p, "sixty") == DASERR_SRC, "sixty is not"); + } + + p = byKey(aList, uCount, "data.lshell.enabled"); + CHECK(p != NULL, "data.lshell.enabled"); + if(p){ + CHECK(DasSettable_type(p) == das_settable_bool, "boolean"); + CHECK(streq(DasSettable_param(p), "read.data"), "flag set parameter"); + CHECK(!DasSettable_required(p), "not required"); + CHECK(quietCheck(p, "yes") == DAS_OKAY, "yes"); + CHECK(quietCheck(p, "OFF") == DAS_OKAY, "OFF"); + CHECK(quietCheck(p, "maybe") == DASERR_SRC, "maybe"); + } + + char sBuf[64]; + p = byKey(aList, uCount, "formats.csv.enabled"); + CHECK(p != NULL, "formats.csv.enabled"); + if(p){ + CHECK(streq(DasSettable_param(p), "format.type"), "format selector param"); + const char* sTitle = DasSettable_title(p, sBuf, sizeof(sBuf)); + CHECK(strncmp(sTitle, "Output as ", 10) == 0, "composed format title, got '%s'", sTitle); + } + + p = byKey(aList, uCount, "formats.csv.delimiter"); + CHECK(p != NULL, "formats.csv.delimiter"); + if(p){ + CHECK(DasSettable_type(p) == das_settable_enum, "delimiter is an enum, got %d", DasSettable_type(p)); + CHECK(DasSettable_enumCount(p) == 4, "four delimiters, got %zu", DasSettable_enumCount(p)); + CHECK(DasSettable_enum(p, 4) == NULL, "past the end"); + CHECK(streq(DasSettable_enum(p, 0), "Comma"), "node spelling kept"); + CHECK(quietCheck(p, DasSettable_enum(p, 0)) == DAS_OKAY, "first choice passes"); + CHECK(quietCheck(p, "tab") == DAS_OKAY, "enum match is case insensitive"); + CHECK(quietCheck(p, "colon") == DASERR_SRC, "colon is not offered"); + } + + p = byKey(aList, uCount, "formats.csv.fracSecs"); + CHECK(p != NULL, "formats.csv.fracSecs"); + if(p){ + const char* sLo = NULL; const char* sHi = NULL; + CHECK(DasSettable_type(p) == das_settable_int, "integer from the parameter"); + CHECK(streq(DasSettable_param(p), "format.secfrac"), "param"); + CHECK(DasSettable_range(p, &sLo, &sHi), "range from the parameter"); + CHECK(streq(sLo, "0") && streq(sHi, "9"), "0 to 9, got %s to %s", sLo, sHi); + CHECK(quietCheck(p, "3") == DAS_OKAY, "3"); + CHECK(quietCheck(p, "12") == DAS_OKAY, "12 passes, ranges are not enforced"); + CHECK(quietCheck(p, "3.5") == DASERR_SRC, "3.5 is not an integer"); + } + + _DasSettable_freeList(aList, uCount); + free(pDom); +} + +static void test_hsk(void) +{ + printf("hsk, current layout at scale:\n"); + DasJdo* pDom = loadJson(HSK); + CHECK(pDom != NULL, "load"); + if(pDom == NULL) return; + + size_t uCount = 0; + DasSettable* aList = parseNode(pDom, &uCount); + CHECK(aList != NULL, "parse"); + CHECK(uCount == 90, "3 + 73 + 14 settables, got %zu", uCount); + + size_t uData = 0; + for(size_t u = 0; u < uCount; ++u) + if(DasSettable_section(aList + u) == das_src_sec_data) ++uData; + CHECK(uData == 73, "73 variable toggles, got %zu", uData); + + /* Every entry has a key, a parameter, and a printable title */ + bool bAllKeyed = true, bAllParam = true, bAllTitled = true; + char sBuf[64]; + for(size_t u = 0; u < uCount; ++u){ + if(DasSettable_key(aList + u)[0] == '\0') bAllKeyed = false; + if(DasSettable_param(aList + u) == NULL) bAllParam = false; + if(DasSettable_title(aList + u, sBuf, sizeof(sBuf)) == NULL) bAllTitled = false; + } + CHECK(bAllKeyed, "all keyed"); + CHECK(bAllParam, "all have a parameter"); + CHECK(bAllTitled, "all titled"); + + const DasSettable* p = byKey(aList, uCount, "coords.time.min"); + CHECK(p != NULL, "coords.time.min"); + if(p){ + const char* sLo = NULL; const char* sHi = NULL; + CHECK(DasSettable_range(p, &sLo, &sHi), "coordinate validRange reaches min"); + CHECK(sLo && (strncmp(sLo, "2025-", 5) == 0), "range low is the mission start, got %s", sLo); + CHECK(streq(DasSettable_title(p, sBuf, sizeof(sBuf)), "Minimum time value to stream"), + "node title wins over the composed one"); + CHECK(streq(DasSettable_units(p), "UTC"), "UTC from the group units"); + } + p = byKey(aList, uCount, "coords.time.res"); + CHECK(p != NULL, "coords.time.res"); + if(p){ + CHECK(!DasSettable_range(p, NULL, NULL), "res has no range"); + CHECK(streq(DasSettable_units(p), "s"), "seconds"); + CHECK(!DasSettable_required(p), "optional"); + } + p = byKey(aList, uCount, "formats.csv.headers"); + CHECK(p != NULL, "formats.csv.headers"); + if(p){ + CHECK(streq(DasSettable_defVal(p), "true"), "default true"); + CHECK(DasSettable_type(p) == das_settable_bool, "boolean"); + } + + _DasSettable_freeList(aList, uCount); + free(pDom); +} + +static void test_juno(void) +{ + printf("juno_survey, 2018 layout:\n"); + DasJdo* pDom = loadJson(JUNO); + CHECK(pDom != NULL, "load"); + if(pDom == NULL) return; + + size_t uCount = 0; + DasSettable* aList = parseNode(pDom, &uCount); + CHECK(aList != NULL, "parse"); + + /* time minimum/maximum/resolution and the options block */ + size_t uCoord = 0, uOpt = 0; + for(size_t u = 0; u < uCount; ++u){ + if(DasSettable_section(aList + u) == das_src_sec_coord) ++uCoord; + if(DasSettable_section(aList + u) == das_src_sec_option) ++uOpt; + } + CHECK(uCoord == 3, "three time settables, got %zu", uCoord); + CHECK(uOpt >= 8, "flag options found, got %zu", uOpt); + CHECK(uCount == uCoord + uOpt, "nothing settable in data, got %zu", uCount); + + /* Keys use the node's long spelling, matching folds it */ + const DasSettable* p = byKey(aList, uCount, "coords.time.minimum"); + CHECK(p != NULL, "coords.time.minimum keyed as spelled"); + CHECK(byKey(aList, uCount, "coords.time.min") == NULL, "not rewritten to min"); + if(p){ + CHECK(_das_src_propEq(DasSettable_prop(p), "min"), "propEq folds it"); + CHECK(streq(DasSettable_param(p), "start_time"), "param"); + CHECK(DasSettable_required(p), "required from the set block"); + CHECK(streq(DasSettable_defVal(p), "2018-10-02T11:00"), "default kept"); + const char* sLo = NULL; const char* sHi = NULL; + CHECK(DasSettable_range(p, &sLo, &sHi) && streq(sHi, "now"), "set range to now"); + CHECK(DasSettable_type(p) == das_settable_time, "type from the old parameter dict"); + CHECK(streq(DasSettable_typeName(p), "isotime"), "type name from the parameter"); + CHECK(quietCheck(p, "2018-10-02T11:00") == DAS_OKAY, "the default passes its own check"); + CHECK(streq(DasSettable_units(p), "UTC"), "UTC from the old units dict"); + char sBuf[64]; + CHECK(streq(DasSettable_title(p, sBuf, sizeof(sBuf)), "Minimum Time value to output"), + "composed from the group name, got '%s'", DasSettable_title(p, sBuf, sizeof(sBuf))); + } + p = byKey(aList, uCount, "coords.time.resolution"); + CHECK(p != NULL, "coords.time.resolution"); + if(p){ + CHECK(streq(DasSettable_defVal(p), "86.4"), "number default in source notation"); + CHECK(streq(DasSettable_units(p), "s"), "units string on the property"); + } + + /* Options: two part keys, empty group, flag toggles are booleans */ + p = byKey(aList, uCount, "options.lfrl"); + CHECK(p != NULL, "options.lfrl"); + if(p){ + CHECK(streq(DasSettable_group(p), ""), "options have no group"); + CHECK(DasSettable_type(p) == das_settable_bool, "flag toggle is boolean"); + CHECK(streq(DasSettable_param(p), "params"), "packed parameter"); + CHECK(streq(DasSettable_defVal(p), "false"), "default false"); + char sBuf[64]; + CHECK(strlen(DasSettable_title(p, sBuf, sizeof(sBuf))) > 10, "node title"); + } + p = byKey(aList, uCount, "options.das2_text"); + CHECK(p != NULL, "options.das2_text"); + if(p) CHECK(DasSettable_type(p) == das_settable_bool, "boolean from a boolean default"); + + _DasSettable_freeList(aList, uCount); + free(pDom); +} + +static void test_edges(void) +{ + printf("Edge cases:\n"); + size_t uCount = 99; + DasSettable* aList = _DasSettable_parse(NULL, NULL, &uCount); + CHECK((aList != NULL) && (uCount == 0), "NULL interface is an empty list"); + _DasSettable_freeList(aList, uCount); + + /* A set block with no param is dropped with a warning, the rest stays */ + const char* sDoc = + "{\"interface\":{\"options\":{" + " \"good\":{\"value\":false,\"set\":{\"param\":\"g\",\"value\":true}}," + " \"broken\":{\"value\":false,\"set\":{\"value\":true}}," + " \"doc_only\":{\"value\":\"x\"}" + "}}}"; + DasJdo* pDom = das_json_parse(sDoc, strlen(sDoc)); + CHECK(pDom != NULL, "parse edge doc"); + daslog_setlevel(DASLOG_ERROR); + aList = _DasSettable_parse(DasJdo_get(pDom, "interface"), NULL, &uCount); + daslog_setlevel(DASLOG_WARN); + CHECK(uCount == 1, "one settable survives, got %zu", uCount); + CHECK((uCount == 1) && streq(DasSettable_key(aList), "options.good"), "the good one"); + CHECK((uCount == 1) && (DasSettable_type(aList) == das_settable_bool), "boolean by default value"); + CHECK((uCount == 1) && streq(DasSettable_title(aList, NULL, 0), ""), "untitled option is empty"); + _DasSettable_freeList(aList, uCount); + free(pDom); + _DasSettable_freeList(NULL, 0); +} + +int main(int argc, char** argv) +{ + das_init(argv[0], DASERR_DIS_EXIT, 0, DASLOG_WARN, NULL); + + test_grammar(); + test_scalar(); + test_geomag(); + test_hsk(); + test_juno(); + test_edges(); + + printf("TestSettable: %d checks passed, %d failed\n", g_nPass, g_nFail); + return (g_nFail == 0) ? 0 : 1; +} From e12774f3f3f6b4e0b436d672a50d59e286b31eb4 Mon Sep 17 00:00:00 2001 From: cpiker Date: Tue, 27 Oct 2026 15:07:42 -0500 Subject: [PATCH 08/18] source: DasSrc, DasReq, and every node has a URI Data sources implemented as standalone objects that provide input validation and input-to-protocol translation facilities. Requisitions added as intentionally simple structured data requests, strings only. Catalog node (and thus data source) ID's clarified as URIs only, all loaded nodes must have some URI, the tag:das2.org,2012:local: prefix is added as a URI of last resort. das_fetch no longer accepts a bare location as -r; a root must have declared URI. settable.h is absorbed into source.h. Developed-with: Claude Fable 5.1 --- .gitignore | 2 +- buildfiles/Darwin.mak | 4 +- buildfiles/Linux.mak | 4 +- buildfiles/Windows.mak | 4 +- buildfiles/cmake/das2CTests.cmake | 2 +- buildfiles/sources.mak | 6 +- das3/catalog.c | 242 ++-- das3/catalog.h | 157 ++- das3/core.h | 3 +- das3/http.c | 24 + das3/http.h | 18 + das3/node.c | 87 +- das3/node.h | 16 +- das3/node_priv.h | 26 + das3/req.c | 527 +++++++++ das3/req.h | 268 +++++ das3/req_priv.h | 87 ++ das3/sett_priv.h | 145 --- das3/settable.c | 537 --------- das3/settable.h | 233 ---- das3/source.c | 1727 +++++++++++++++++++++++++++++ das3/source.h | 635 +++++++++++ das3/src_priv.h | 163 +++ das3/time.c | 102 +- das3/time.h | 24 +- test/TestCat.c | 102 +- test/TestSettable.c | 415 ------- test/TestSource.c | 810 ++++++++++++++ test/das_fetch/cat/site.json | 2 +- test/das_fetch_test.sh | 20 +- utilities/das_fetch.c | 81 +- 31 files changed, 4906 insertions(+), 1567 deletions(-) create mode 100644 das3/req.c create mode 100644 das3/req.h create mode 100644 das3/req_priv.h delete mode 100644 das3/sett_priv.h delete mode 100644 das3/settable.c delete mode 100644 das3/settable.h create mode 100644 das3/source.c create mode 100644 das3/source.h create mode 100644 das3/src_priv.h delete mode 100644 test/TestSettable.c create mode 100644 test/TestSource.c diff --git a/.gitignore b/.gitignore index 35b87b7..969705e 100644 --- a/.gitignore +++ b/.gitignore @@ -12,5 +12,5 @@ test/test_nan tmp/* CLAUDE.md .claude -co_notes +c*_notes diff --git a/buildfiles/Darwin.mak b/buildfiles/Darwin.mak index 64ea503..1c6f668 100644 --- a/buildfiles/Darwin.mak +++ b/buildfiles/Darwin.mak @@ -376,8 +376,8 @@ test_main: $(BD) $(BD)/$(TARG).a $(BUILD_TEST_PROGS) $(BULID_UTIL_PROGS) @$(BD)/TestCredMngr $(BD) @echo "INFO: Running unit test for catalog sessions over local nodes, $(BD)/TestCat..." @$(BD)/TestCat - @echo "INFO: Running unit test for source settings over local nodes, $(BD)/TestSettable..." - @$(BD)/TestSettable + @echo "INFO: Running unit test for source descriptions and requisitions, $(BD)/TestSource..." + @$(BD)/TestSource @echo "INFO: Running unit test for the URL parser, $(BD)/TestHttp..." @$(BD)/TestHttp @echo "INFO: Running unit test for exception names and send.h writers, $(BD)/TestOob..." diff --git a/buildfiles/Linux.mak b/buildfiles/Linux.mak index 0880c0a..a02f695 100644 --- a/buildfiles/Linux.mak +++ b/buildfiles/Linux.mak @@ -374,8 +374,8 @@ test_main: $(BD) $(BD)/$(TARG).a $(BUILD_TEST_PROGS) $(BUILD_UTIL_PROGS) @$(BD)/TestCredMngr $(BD) @echo "INFO: Running unit test for catalog sessions over local nodes, $(BD)/TestCat..." @$(BD)/TestCat - @echo "INFO: Running unit test for source settings over local nodes, $(BD)/TestSettable..." - @$(BD)/TestSettable + @echo "INFO: Running unit test for source descriptions and requisitions, $(BD)/TestSource..." + @$(BD)/TestSource @echo "INFO: Running unit test for the URL parser, $(BD)/TestHttp..." @$(BD)/TestHttp @echo "INFO: Running unit test for exception names and send.h writers, $(BD)/TestOob..." diff --git a/buildfiles/Windows.mak b/buildfiles/Windows.mak index 8c61f00..d8e067a 100644 --- a/buildfiles/Windows.mak +++ b/buildfiles/Windows.mak @@ -390,8 +390,8 @@ test_main: @$(BD)\TestCredMngr.exe $(BD) @echo INFO: Running unit test for catalog sessions over local nodes, $(BD)\TestCat... @$(BD)\TestCat.exe - @echo INFO: Running unit test for source settings over local nodes, $(BD)\TestSettable... - @$(BD)\TestSettable.exe + @echo INFO: Running unit test for source descriptions and requisitions, $(BD)\TestSource... + @$(BD)\TestSource.exe @echo INFO: Running unit test for the URL parser, $(BD)\TestHttp... @$(BD)\TestHttp.exe @echo INFO: Running unit test for exception names and send.h writers, $(BD)\TestOob... diff --git a/buildfiles/cmake/das2CTests.cmake b/buildfiles/cmake/das2CTests.cmake index b73b082..fca8f2f 100644 --- a/buildfiles/cmake/das2CTests.cmake +++ b/buildfiles/cmake/das2CTests.cmake @@ -65,7 +65,7 @@ _das_test(TestFilter main TestFilter) _das_test(TestUri main TestUri "${_bd}/uri_tplt") _das_test(TestCredMngr main TestCredMngr "${_bd}") _das_test(TestCat main TestCat) -_das_test(TestSettable main TestSettable) +_das_test(TestSource main TestSource) _das_test(TestHttp main TestHttp) _das_test(TestOob main TestOob) foreach(_v 2 3) diff --git a/buildfiles/sources.mak b/buildfiles/sources.mak index b8b2647..440d19c 100644 --- a/buildfiles/sources.mak +++ b/buildfiles/sources.mak @@ -15,7 +15,7 @@ LIB_SRCS = das3/das1.c das3/array.c das3/buffer.c das3/builder.c das3/cli.c \ das3/dataset_hdr2.c das3/dataset_hdr3.c das3/datum.c das3/descriptor.c \ das3/dft.c das3/dimension.c das3/dsdf.c das3/encoding.c das3/http.c \ das3/io.c das3/iterator.c das3/json.c das3/toml.c das3/log.c das3/node.c \ - das3/catalog.c das3/settable.c \ + das3/catalog.c das3/req.c das3/source.c \ das3/oob.c das3/operator.c das3/packet.c das3/plane.c das3/processor.c \ das3/property.c das3/send.c das3/stream.c das3/time.c das3/tt2000.c \ das3/units.c das3/utf8.c das3/util.c das3/value.c \ @@ -30,7 +30,7 @@ LIB_HDRS = das3/defs.h das3/time.h das3/das1.h das3/util.h das3/log.h \ das3/dimension.h das3/dataset.h das3/plane.h das3/packet.h das3/stream.h \ das3/processor.h das3/property.h das3/oob.h das3/io.h das3/iterator.h \ das3/builder.h das3/dsdf.h das3/credentials.h das3/http.h das3/dft.h \ - das3/json.h das3/toml.h das3/node.h das3/catalog.h das3/settable.h das3/cli.h das3/send.h das3/uri.h \ + das3/json.h das3/toml.h das3/node.h das3/catalog.h das3/req.h das3/source.h das3/cli.h das3/send.h das3/uri.h \ das3/codec.h das3/codex.h das3/core.h das3/variable.h \ das3/form.h das3/form_rot.h das3/form_lin.h das3/form_pt.h \ das3/form_cplx.h das3/form_vec.h das3/form_pos.h das3/das2_compat.h \ @@ -61,7 +61,7 @@ CDF_UTIL_SRCS = utilities/das3_cdf.c utilities/das3_from_cdf.c \ utilities/das2_from_cdf.c TEST_SRCS = test/TestUnits.c test/TestArray.c test/TestDs.c test/TestBuilder.c \ - test/TestAuth.c test/TestCatalog.c test/TestCat.c test/TestSettable.c test/TestHttp.c test/TestTT2000.c test/ex_das_cli.c \ + test/TestAuth.c test/TestCatalog.c test/TestCat.c test/TestSource.c test/TestHttp.c test/TestTT2000.c test/ex_das_cli.c \ test/ex_das_ephem.c test/TestCredMngr.c test/TestV3Read.c test/TestFeed.c \ test/TestProp.c \ test/TestIter.c test/TestUri.c test/TestFilter.c test/TestValue.c \ diff --git a/das3/catalog.c b/das3/catalog.c index 27b1c74..4528dcc 100644 --- a/das3/catalog.c +++ b/das3/catalog.c @@ -32,6 +32,7 @@ #include "util.h" #include "log.h" #include "node_priv.h" +#include "src_priv.h" #include "catalog.h" struct das_cat_mount { @@ -45,16 +46,21 @@ struct das_cat { DasNodeCache* pCache; /* owned, NULL until DasCat_setCache() */ /* Root policy. Relative paths start at sPrefix in the federated - * catalog, unless a root is set, then at that node, or with mounts, at - * the first mounted node, or at the prefix if one was given. */ + * catalog, or with mounts at the root node if one was set, else at + * the first mounted node, or at the prefix if one was given. + * + * A root is a mount that turns off fall through: the mounts are then + * the whole catalog. Keeping one list means a root and further mounts + * mix freely, which is how an offline test catalog is assembled from a + * handful of files. */ char* sPrefix; /* never NULL */ bool bPrefixGiven; - struct das_cat_mount root; /* the whole catalog; sUrl NULL if unset, - sUri NULL for a nameless root */ - struct das_cat_mount* aMounts; /* over the federated roots */ + struct das_cat_mount* aMounts; /* named nodes loaded by location */ size_t uMounts; + bool bIsolated; /* set by DasCat_setRoot(): no fall through */ + size_t uRoot; /* index of the root in aMounts when isolated */ - DasNode* pRoot; /* memory root of the root and mount modes */ + DasNode* pRoot; /* the mount root, built on first use */ DasNode** aRoots; /* every memory root made, freed with the session */ size_t uRoots; size_t uRootCap; @@ -77,14 +83,21 @@ static void _setStr(char** ps, const char* s) *ps = (s != NULL) ? das_strdup(s) : NULL; } -/* site:/x and test:/x are shorthand for the two well known children - * of the das2 root. new_RootNode() expands them too; doing it here as well - * keeps DasCat_lastUri() the full form. */ +/* Shorthands are expanded by node.c as well; doing it here too keeps + * DasCat_lastUri() the full form even when a resolve fails early. */ static char* _expand(const char* sIn) { - if(_pfx(sIn, "site:")||_pfx(sIn, "test:")) - return das_string(D2URI_ROOT "%s", sIn); - return das_strdup(sIn); + char sBuf[600]; + return das_strdup(_das_uri_expand(sIn, sBuf, sizeof(sBuf))); +} + +bool das_isUri(const char* sPath) +{ + if((sPath == NULL) || !isalpha((unsigned char)sPath[0])) return false; + const char* p = sPath + 1; + while(isalnum((unsigned char)*p) || (*p == '+') || (*p == '-') || (*p == '.')) ++p; + /* A single letter before the colon is a Windows drive, not a scheme */ + return (*p == ':') && (p > sPath + 1); } bool das_isUrlOrFile(const char* sPath) @@ -156,8 +169,6 @@ void del_DasCat(DasCat* pThis) free(pThis->sAgent); free(pThis->sPrefix); - free(pThis->root.sUrl); - free(pThis->root.sUri); for(size_t u = 0; u < pThis->uMounts; ++u){ free(pThis->aMounts[u].sUrl); free(pThis->aMounts[u].sUri); @@ -229,13 +240,57 @@ DasErrCode DasCat_setPrefix(DasCat* pThis, const char* sUri) } free(pThis->sPrefix); pThis->sPrefix = _expand(sUri); + /* A prefix is joined to relative paths by plain concatenation, so it + must end in the container's separator. A bare shorthand names the + container; its separator is known, supply it. */ + if((strcmp(pThis->sPrefix, D2URI_SITES) == 0)||(strcmp(pThis->sPrefix, D2URI_ROOT "test") == 0)){ + char* s = das_string("%s:/", pThis->sPrefix); + free(pThis->sPrefix); + pThis->sPrefix = s; + } pThis->bPrefixGiven = true; return DAS_OKAY; } const char* DasCat_prefix(const DasCat* pThis){ return pThis->sPrefix; } -bool DasCat_hasRoot(const DasCat* pThis){ return pThis->root.sUrl != NULL; } +bool DasCat_hasRoot(const DasCat* pThis){ return pThis->bIsolated; } + +/* The mount whose URI covers sUri, or -1. Longest match, as the mount + * root itself resolves, though mounts rarely nest. */ +static int _DasCat_mountFor(const DasCat* pThis, const char* sUri) +{ + int nBest = -1; + size_t uBest = 0; + for(size_t u = 0; u < pThis->uMounts; ++u){ + size_t uLen = strlen(pThis->aMounts[u].sUri); + if((uLen > uBest) && (strncmp(sUri, pThis->aMounts[u].sUri, uLen) == 0)){ + nBest = (int)u; + uBest = uLen; + } + } + return nBest; +} + +/* Add to the mount list. An empty URI names the node by its location. */ +static DasErrCode _DasCat_addMount(DasCat* pThis, const char* sUrl, const char* sUri) +{ + struct das_cat_mount* a = (struct das_cat_mount*)realloc( + pThis->aMounts, (pThis->uMounts + 1) * sizeof(struct das_cat_mount) + ); + if(a == NULL) return das_error(DASERR_CAT, "Couldn't grow the mount list"); + pThis->aMounts = a; + a[pThis->uMounts].sUrl = das_strdup(sUrl); + if((sUri == NULL)||(sUri[0] == '\0')){ + char sBuf[600]; + a[pThis->uMounts].sUri = das_strdup(_das_uri_local(sUrl, sBuf, sizeof(sBuf))); + } + else{ + a[pThis->uMounts].sUri = _expand(sUri); + } + pThis->uMounts += 1; + return DAS_OKAY; +} /* The checks shared by roots and mounts */ static bool _DasCat_locationOk(const char* sUrl, const char* sUri, bool bUriNeeded) @@ -259,43 +314,32 @@ static bool _DasCat_locationOk(const char* sUrl, const char* sUri, bool bUriNeed DasErrCode DasCat_setRoot(DasCat* pThis, const char* sUrl, const char* sUri) { if(_DasCat_frozen(pThis)) return DASERR_CAT; - if(DasCat_hasRoot(pThis)){ + if(pThis->bIsolated){ daslog_error("A catalog has one root, and one is already set"); return DASERR_CAT; } - if(pThis->uMounts > 0){ - daslog_error("A root replaces the federated catalog, which the mounts build on; use one or the other"); - return DASERR_CAT; - } if(!_DasCat_locationOk(sUrl, sUri, false)) return DASERR_CAT; - _setStr(&pThis->root.sUrl, sUrl); - free(pThis->root.sUri); - pThis->root.sUri = ((sUri != NULL) && (sUri[0] != '\0')) ? _expand(sUri) : NULL; + if(_DasCat_addMount(pThis, sUrl, sUri) != DAS_OKAY) return DASERR_CAT; + pThis->uRoot = pThis->uMounts - 1; + pThis->bIsolated = true; return DAS_OKAY; } -const char* DasCat_rootUrl(const DasCat* pThis){ return pThis->root.sUrl; } +const char* DasCat_rootUrl(const DasCat* pThis) +{ + return pThis->bIsolated ? pThis->aMounts[pThis->uRoot].sUrl : NULL; +} -const char* DasCat_rootUri(const DasCat* pThis){ return pThis->root.sUri; } +const char* DasCat_rootUri(const DasCat* pThis) +{ + return pThis->bIsolated ? pThis->aMounts[pThis->uRoot].sUri : NULL; +} DasErrCode DasCat_mount(DasCat* pThis, const char* sUrl, const char* sUri) { if(_DasCat_frozen(pThis)) return DASERR_CAT; - if(DasCat_hasRoot(pThis)){ - daslog_error("A mount builds on the federated catalog, which a root replaces; use one or the other"); - return DASERR_CAT; - } if(!_DasCat_locationOk(sUrl, sUri, true)) return DASERR_CAT; - - struct das_cat_mount* a = (struct das_cat_mount*)realloc( - pThis->aMounts, (pThis->uMounts + 1) * sizeof(struct das_cat_mount) - ); - if(a == NULL) return das_error(DASERR_CAT, "Couldn't grow the mount list"); - pThis->aMounts = a; - a[pThis->uMounts].sUrl = das_strdup(sUrl); - a[pThis->uMounts].sUri = _expand(sUri); - pThis->uMounts += 1; - return DAS_OKAY; + return _DasCat_addMount(pThis, sUrl, sUri); } size_t DasCat_mountCount(const DasCat* pThis){ return pThis->uMounts; } @@ -303,47 +347,42 @@ size_t DasCat_mountCount(const DasCat* pThis){ return pThis->uMounts; } /* ************************************************************************* */ /* Resolution */ -/* Root node of the root and mount modes, loaded on first use. A named - * root is a mount root with one mount and no fall through. */ +/* The mount root, loaded on first use. With a root set the mounts are the + * whole catalog, otherwise unclaimed paths fall through to the federated + * roots. */ static DasNode* _DasCat_root(DasCat* pThis) { if(pThis->pRoot != NULL) return pThis->pRoot; + if(pThis->uMounts == 0) return NULL; - if(pThis->root.sUrl != NULL){ - if(pThis->root.sUri == NULL){ - pThis->pRoot = _DasNode_newRootUrl( - pThis->root.sUrl, "", pThis->pMgr, pThis->sAgent, pThis->pCache - ); - } - else{ - const char* aUrl[1] = { pThis->root.sUrl }; - const char* aUri[1] = { pThis->root.sUri }; - pThis->pRoot = _DasNode_newMountRoot( - aUrl, aUri, 1, false, pThis->pMgr, pThis->sAgent, pThis->pCache - ); - } + const char** aUrl = (const char**)calloc(pThis->uMounts, sizeof(char*)); + const char** aUri = (const char**)calloc(pThis->uMounts, sizeof(char*)); + if((aUrl == NULL)||(aUri == NULL)){ + das_error(DASERR_CAT, "Couldn't allocate the mount root"); + free(aUrl); free(aUri); + return NULL; } - else if(pThis->uMounts > 0){ - const char** aUrl = (const char**)calloc(pThis->uMounts, sizeof(char*)); - const char** aUri = (const char**)calloc(pThis->uMounts, sizeof(char*)); - if((aUrl == NULL)||(aUri == NULL)){ - das_error(DASERR_CAT, "Couldn't allocate the mount root"); - free(aUrl); free(aUri); - return NULL; - } - for(size_t u = 0; u < pThis->uMounts; ++u){ - aUrl[u] = pThis->aMounts[u].sUrl; - aUri[u] = pThis->aMounts[u].sUri; - } - pThis->pRoot = _DasNode_newMountRoot( - aUrl, aUri, pThis->uMounts, true, pThis->pMgr, pThis->sAgent, pThis->pCache - ); - free(aUrl); - free(aUri); + for(size_t u = 0; u < pThis->uMounts; ++u){ + aUrl[u] = pThis->aMounts[u].sUrl; + aUri[u] = pThis->aMounts[u].sUri; } + pThis->pRoot = _DasNode_newMountRoot( + aUrl, aUri, pThis->uMounts, !pThis->bIsolated, pThis->pMgr, pThis->sAgent, + pThis->pCache + ); + free(aUrl); + free(aUri); return _DasCat_keep(pThis, pThis->pRoot); } +/* A local: name reaches only the session's own mounts, see D2URI_LOCAL */ +static bool _DasCat_unrouted(const DasCat* pThis, const char* sUri) +{ + if(!_das_uri_isLocal(sUri) || (_DasCat_mountFor(pThis, sUri) >= 0)) return false; + daslog_debug_v("%s is a local: name and nothing is mounted there", sUri); + return true; +} + DasNode* DasCat_resolve(DasCat* pThis, const char* sPath) { bool bNoPath = (sPath == NULL)||(sPath[0] == '\0'); @@ -351,35 +390,36 @@ DasNode* DasCat_resolve(DasCat* pThis, const char* sPath) free(pThis->sLastUri); pThis->sLastUri = NULL; - /* A node by location, named by itself */ + /* A node by location, named under local: */ if(!bNoPath && das_isUrlOrFile(sPath)){ - pThis->sLastUri = das_strdup(sPath); + char sUri[600]; + pThis->sLastUri = das_strdup(_das_uri_local(sPath, sUri, sizeof(sUri))); return _DasCat_keep(pThis, _DasNode_newRootUrl( - sPath, sPath, pThis->pMgr, pThis->sAgent, pThis->pCache + sPath, pThis->sLastUri, pThis->pMgr, pThis->sAgent, pThis->pCache )); } - const char* sRootUri = pThis->root.sUri; - bool bMounted = (pThis->uMounts > 0); - if((sRootUri != NULL) || bMounted){ + if(pThis->uMounts > 0){ DasNode* pRoot = _DasCat_root(pThis); if(pRoot == NULL) return NULL; - const char* sTopUri = bMounted ? pThis->aMounts[0].sUri : sRootUri; + const char* sTopUri = pThis->aMounts[pThis->bIsolated ? pThis->uRoot : 0].sUri; - /* No path: the named node itself. A full URI, or a relative path - * under an explicit prefix when the catalog is mounted over: the - * mount root decides whether a mount or the federated catalog owns - * it. Anything else is relative to the named node. */ + /* No path: the top node itself. A full URI, or a relative path + * under an explicit prefix: the mount root decides whether a mount + * or the federated catalog owns it. Anything else is relative to + * the top node. */ if(bNoPath){ pThis->sLastUri = das_strdup(sTopUri); return DasNode_subNode(pRoot, sTopUri, NULL, NULL); } - if(bMounted && pThis->bPrefixGiven) + if(pThis->bPrefixGiven && !das_isUri(sPath)) pThis->sLastUri = das_string("%s%s", pThis->sPrefix, sPath); else pThis->sLastUri = _expand(sPath); - if(_pfx(pThis->sLastUri, "tag:")||_pfx(pThis->sLastUri, sTopUri)) + if(das_isUri(pThis->sLastUri)){ + if(_DasCat_unrouted(pThis, pThis->sLastUri)) return NULL; return DasNode_subNode(pRoot, pThis->sLastUri, NULL, NULL); + } DasNode* pTop = DasNode_subNode(pRoot, sTopUri, NULL, NULL); if(pTop == NULL){ @@ -392,17 +432,6 @@ DasNode* DasCat_resolve(DasCat* pThis, const char* sPath) return DasNode_subNode(pTop, sPath, NULL, NULL); } - if(pThis->root.sUrl != NULL){ - DasNode* pRoot = _DasCat_root(pThis); - if(pRoot == NULL) return NULL; - if(bNoPath){ - pThis->sLastUri = das_strdup(pThis->root.sUrl); - return pRoot; - } - pThis->sLastUri = das_strdup(sPath); - return DasNode_subNode(pRoot, sPath, NULL, NULL); - } - /* The federated catalog under the prefix */ if(bNoPath){ /* The prefix with its trailing separator removed names the container */ @@ -411,11 +440,12 @@ DasNode* DasCat_resolve(DasCat* pThis, const char* sPath) while((u > 0) && ((pThis->sLastUri[u-1] == '/')||(pThis->sLastUri[u-1] == ':'))) pThis->sLastUri[--u] = '\0'; } - else if(_pfx(sPath, "tag:")||_pfx(sPath, "site:")||_pfx(sPath, "test:")) + else if(das_isUri(sPath)) pThis->sLastUri = _expand(sPath); else pThis->sLastUri = das_string("%s%s", pThis->sPrefix, sPath); + if(_DasCat_unrouted(pThis, pThis->sLastUri)) return NULL; return _DasCat_keep(pThis, _DasNode_newRoot( pThis->sLastUri, pThis->pMgr, pThis->sAgent, pThis->pCache )); @@ -425,10 +455,10 @@ const char* DasCat_lastUri(const DasCat* pThis){ return pThis->sLastUri; } bool DasCat_covers(const DasCat* pThis, const char* sUri) { - /* Mounts fall through to the federated catalog and a nameless root has - * no URIs to be outside of, only a named root walls paths off */ - if(pThis->root.sUri == NULL) return true; - return _pfx(sUri, pThis->root.sUri); + /* local: names go nowhere but a mount; otherwise mounts fall through to + * the federated catalog and only a root walls paths off */ + if(_das_uri_isLocal(sUri) || pThis->bIsolated) return _DasCat_mountFor(pThis, sUri) >= 0; + return true; } DasNode* DasCat_nearest(DasCat* pThis, const char** psRest) @@ -473,3 +503,15 @@ DasNode* DasCat_nearest(DasCat* pThis, const char** psRest) } return pNode; } + +/* ************************************************************************* */ +/* Sources */ + +DasSrc* DasCat_newSrc(DasCat* pThis, const char* sPath) +{ + DasNode* pNode = DasCat_resolve(pThis, sPath); + if(pNode == NULL) return NULL; + DasNode* pSrcNode = _DasSrc_defNode(pNode); + if(pSrcNode == NULL) return NULL; + return new_DasSrc_node(pSrcNode); +} diff --git a/das3/catalog.h b/das3/catalog.h index 4dac9c1..816877e 100644 --- a/das3/catalog.h +++ b/das3/catalog.h @@ -22,38 +22,62 @@ #define _das_catalog_h_ #include +#include #ifdef __cplusplus extern "C" { #endif /** @defgroup catalog Catalogs - * Finding data sources: the federated das catalog, catalog paths, and the + * Finding data sources: the federated das catalog, catalog URIs, and the * objects that walk it. * * A das catalog is a tree of small JSON documents, mostly on web servers, * leading from a well known root down to data source definitions. Each * document is a node. Container nodes (Catalog, Collection, SourceSet) - * list their children by name along with the URLs each child can be loaded - * from. Leaf nodes (HttpStreamSrc, FileTimeAgg) describe how to get data. - * The tree is federated: the root at das2.org names a few top level trees - * and each site maintains its own branch. A path is resolved one network - * request per level, and only the levels on the way to the node asked for - * are ever loaded. + * list their children by name along with the URLs each child can be + * loaded from. Leaf nodes (HttpStreamSrc, FileTimeAgg) describe how to + * get data. The tree is federated: the root at das2.org names a few top + * level trees, and any node may have children maintained by a different + * organization. A path is resolved one network request per level, and + * only the levels on the way to the node asked for are ever loaded. * - * Nodes are named by catalog path, which looks like a file path with a URI - * scheme in front: + * @par Names + * + * Every node has a globally unique URI, composed by walking down from the + * root: each container appends its separator and the child's name to its + * own URI. A node does not know its name, the walk gives it one. The + * result looks like a file path with a URI scheme in front: * * tag:das2.org,2012:site:/uiowa/juno/wav/survey * - * In most programs the "tag:das2.org,2012:site:/" prefix is noise, so the + * The RFC 4151 "tag" scheme is the usual one, though others such as "doi" + * work just as well. URLs are not used as names because they change + * whenever an organization reshuffles its servers, while a name must not. + * For that reason a name may not start with a locator scheme, "http", + * "https", "ftp" and so on; the library reads those as locations, see + * das_isUrlOrFile(). + * + * In daily use the prefix "tag:das2.org,2012:site:/" is noise, so the * library also accepts the relative form "uiowa/juno/wav/survey" and the - * shorthand "site:/uiowa/juno/wav/survey". A node can also be - * named by location, the URL or file it is stored at, which is how test - * catalogs and development servers are reached. + * shorthand "site:/uiowa/juno/wav/survey". + * + * Since names come from the walk, a local file can be declared to be any + * node. For testing, the file + * + * ./survey.json + * + * can be mounted as * - * To get a data source definition, open a catalog session, resolve a path - * and read the node: + * tag:das2.org,2012:site:/uiowa/juno/wav/survey + * + * and walking proceeds from there as if the file had been fetched from + * the catalog, see DasCat_mount() and DasCat_setRoot(). A file loaded + * with no name at all is named after its location under D2URI_LOCAL. + * + * @par Reading a node + * + * Open a catalog session, resolve a path and read the node: * * @code DasCat* pCat = new_DasCat(NULL, "my_program/1.0"); @@ -67,15 +91,19 @@ extern "C" { del_DasCat(pCat); // frees every node the session loaded @endcode * + * Programs that want to query a data source rather than inspect its node + * use DasCat_newSrc(), which resolves the path and builds a DasSrc, see + * source.h. + * * Two objects do the work. DasCat, in das3/catalog.h, is a session: it * holds the root policy (where relative paths start, see DasCat_setRoot() - * and DasCat_mount()), the credentials and user agent every load uses, and - * the memory of every node it loads. DasNode, in das3/node.h, is one node: - * its type, path URI, source URL and parsed document, with DasNode_subNode() - * to step down to a child. Most programs only ever construct a DasCat. - * The node constructors in node.h are the lower level interface for - * programs that manage roots themselves; see "Working with nodes directly" - * below. + * and DasCat_mount()), the credentials and user agent every load uses, + * and the memory of every node it loads. DasNode, in das3/node.h, is one + * node: its type, URI, source URL and parsed document, with + * DasNode_subNode() to step down to a child. Most programs only ever + * construct a DasCat. The node constructors in node.h are the lower + * level interface for programs that manage roots themselves; see + * "Working with nodes directly" in node.h. */ /** @addtogroup catalog @@ -102,21 +130,22 @@ extern "C" { * - the shorthands "site:/uiowa/..." and "test:/..." * - a full catalog URI: "tag:das2.org,2012:site:/uiowa/..." * - a node by location, a URL or a file: "https://host/node.json", - * "./node.json" + * "./node.json". Such a node is named after its location under + * D2URI_LOCAL, since every node has a URI. * * Where relative paths start is the root policy, and the catalog tree is * navigated like a file system, so the policy borrows its words: * * - By default relative paths start at DASCAT_DEF_PREFIX in the * federated catalog. DasCat_setPrefix() moves that. - * - DasCat_setRoot() makes one node, loaded by location, the whole - * catalog. Optionally the node is given its catalog URI, so paths - * under that URI resolve through it and DasNode_pathUri() reports - * full URIs. * - DasCat_mount() puts a node, loaded by location, at a catalog URI * on top of the federated catalog. Paths under the URI resolve * through the mounted node, everything else through the compiled in * roots as usual. Several nodes may be mounted. + * - DasCat_setRoot() mounts a node and makes the mounts the whole + * catalog: relative paths start at that node and nothing outside + * the mounts can be reached. This is how an offline catalog is + * assembled from a few files. * * Policy is set before the first resolve. Nodes returned by * DasCat_resolve() are owned by the catalog and live until del_DasCat(). @@ -145,6 +174,18 @@ typedef struct das_cat DasCat; */ DAS_API bool das_isUrlOrFile(const char* sPath); +/** Is this string a full URI rather than a relative catalog path? + * + * True when it starts with a URI scheme, a letter followed by letters, + * digits, '+', '-' or '.' and then a colon, as in "tag:", "doi:" and the + * shorthands "site:" and "test:". A single letter before the colon is a + * Windows drive, not a scheme. Note that URLs are URIs too; test with + * das_isUrlOrFile() first when the difference matters. + * + * @memberof DasCat + */ +DAS_API bool das_isUri(const char* sPath); + /** Create a catalog session * * The returned catalog it's node cache enabled right out of the gate, @@ -214,7 +255,9 @@ DAS_API int DasCat_cacheAge(const DasCat* pThis); /** Where relative paths start in the federated catalog * - * The shorthands "site:" and "test:" are expanded. A URL or file is + * The shorthands "site:" and "test:" are expanded; given alone they name + * the container and its ":/" child separator is supplied, so "site:" is + * the same prefix as "tag:das2.org,2012:site:/". A URL or file is * refused, those are roots, see DasCat_setRoot(). * * @param sUri A catalog URI prefix, typically ending in the separator the @@ -229,22 +272,24 @@ DAS_API DasErrCode DasCat_setPrefix(DasCat* pThis, const char* sUri); */ DAS_API const char* DasCat_prefix(const DasCat* pThis); -/** Make one node, loaded by location, the whole catalog +/** Mount a node and make the mounts the whole catalog * - * Relative paths then start at that node and nothing outside it can be - * reached. Refused if a root is already set or anything is mounted. + * Relative paths then start at this node, full URIs resolve only when a + * mount covers them, and everything else fails, which DasCat_covers() can + * tell apart from an ordinary miss. Further DasCat_mount() calls, before + * or after, add to the offline catalog. Refused once a root is set. * - * Without a URI the root has no catalog name: DasNode_pathUri() on nodes - * under it gives paths relative to the root, and a full URI given to - * DasCat_resolve() is just another relative path. With a URI the root is - * declared to be the node of that name, as a development server's copy of - * a catalog branch stands for the branch's federated name. Then full URIs - * under it resolve through it, nodes report full URIs, and URIs outside it - * fail, which DasCat_covers() can tell apart from an ordinary miss. + * The URI declares what the node is: a development server's copy of a + * catalog branch stands for the branch's federated name, so requisitions + * made against it are good against the live catalog too. Without a URI + * the node is named after its location under D2URI_LOCAL. That is fine + * for a one off file, but a requisition stamped with a local: name will + * not match the catalog node the file was copied from; give the real URI + * when it has one. * * @param sUrl A URL or file path that new_RootNode_url() accepts * @param sUri The catalog URI the node has, shorthands accepted, or NULL - * for a nameless root + * to name it by location * @returns DAS_OKAY, or DASERR_CAT with the reason logged * @memberof DasCat */ @@ -267,7 +312,8 @@ DAS_API const char* DasCat_rootUri(const DasCat* pThis); * roots as usual. May be called more than once; when one mount point is * under another the longer URI wins. Relative paths start at the first * mount unless DasCat_setPrefix() was called, then they start at the - * prefix. Refused if a root is set. + * prefix. With a root set, see DasCat_setRoot(), the mount joins the + * offline catalog instead. * * @param sUrl Where to load the node from, a URL or a file * @param sUri The catalog URI to mount it at, shorthands accepted @@ -281,7 +327,7 @@ DAS_API DasErrCode DasCat_mount(DasCat* pThis, const char* sUrl, const char* sUr */ DAS_API size_t DasCat_mountCount(const DasCat* pThis); -/** True if a root is set, so paths no longer reach the whole federated catalog +/** True if a root is set, so paths reach only the mounted nodes * @memberof DasCat */ DAS_API bool DasCat_hasRoot(const DasCat* pThis); @@ -311,9 +357,11 @@ DAS_API const char* DasCat_lastUri(const DasCat* pThis); /** Can this URI be reached under the current policy? * - * False only when a named root is set and sUri is not under its URI. A - * failed resolve of a URI this returns false for is a policy miss, not a - * catalog miss, and the message to the user should say so. + * False when a root is set and no mount covers sUri, and for a local: + * name, see D2URI_LOCAL, that no mount covers; those are never looked up + * in the federated catalog. A failed resolve of a URI this returns false + * for is a policy miss, not a catalog miss, and the message to the user + * should say so. * @memberof DasCat */ DAS_API bool DasCat_covers(const DasCat* pThis, const char* sUri); @@ -335,6 +383,27 @@ DAS_API bool DasCat_covers(const DasCat* pThis, const char* sUri); */ DAS_API DasNode* DasCat_nearest(DasCat* pThis, const char** psRest); +/** Resolve a path and build the data source it leads to. + * + * Catalog paths usually end at a Collection or SourceSet whose children + * are the actual sources, often one per server interface, and users + * rarely type the child's name. This resolves the path as + * DasCat_resolve() does and then picks: the node itself if it is + * already a stream source, else the child named "flex", which is the + * current server interface, else the first stream source child in the + * container's order. The source is built from that node with + * new_DasSrc_node(). + * + * @param sPath A catalog path in any form DasCat_resolve() accepts, + * including a URL or file name. + * @returns A new source, to be freed with del_DasSrc() and independent + * of this session afterwards, or NULL after logging why not. As + * with a failed resolve, DasCat_lastUri() and DasCat_nearest() + * say how far the path got. + * @memberof DasCat + */ +DAS_API DasSrc* DasCat_newSrc(DasCat* pThis, const char* sPath); + /** @} */ #ifdef __cplusplus diff --git a/das3/core.h b/das3/core.h index 71ca0c0..ec4d582 100644 --- a/das3/core.h +++ b/das3/core.h @@ -233,7 +233,8 @@ das3/utf8.h Counting characters instead of bytes das3/toml.h Reading TOML configuration files das3/dft.h Properly scaled power spectral densities das3/catalog.h Finding data sources in the federated catalog system -das3/settable.h The adjustable properties of a data source +das3/source.h Data source descriptions, their properties and controls +das3/req.h Requisitions, what a client wants from a data source das3/http.h Reading streams from servers, credentials.h handles logins das3/builder.h Collect a whole das2 or das3 stream into DasDs objects das3/packet.h das2 packet descriptors, new code should use DasDs instead diff --git a/das3/http.c b/das3/http.c index 6be35fc..e407afe 100644 --- a/das3/http.c +++ b/das3/http.c @@ -383,6 +383,30 @@ char* das_url_toStr(const struct das_url* pUrl, char* sBuf, size_t uLen) return _prn_fit(sBuf, uLen, n); } +size_t das_url_encode(const char* sIn, char* sOut, size_t uOut) +{ + static const char* sHex = "0123456789ABCDEF"; + size_t uNeed = 0; + if(uOut > 0) sOut[0] = '\0'; + for(const unsigned char* p = (const unsigned char*)sIn; p && *p; ++p){ + bool bPlain = isalnum(*p)||(*p == '-')||(*p == '.')||(*p == '_')||(*p == '~'); + size_t n = bPlain ? 1 : 3; + if(uNeed + n < uOut){ + if(bPlain){ + sOut[uNeed] = (char)*p; + } + else{ + sOut[uNeed] = '%'; + sOut[uNeed + 1] = sHex[*p >> 4]; + sOut[uNeed + 2] = sHex[*p & 0x0F]; + } + sOut[uNeed + n] = '\0'; + } + uNeed += n; + } + return uNeed; +} + /* ************************************************************************** */ /* HTTP Client Functions */ diff --git a/das3/http.h b/das3/http.h index 435d727..6d6cb23 100644 --- a/das3/http.h +++ b/das3/http.h @@ -124,6 +124,24 @@ typedef struct das_http_response_t{ */ DAS_API char* das_url_toStr(const struct das_url* pUrl, char* sBuf, size_t uLen); +/** Percent encode a string for use in a URL query. + * + * Everything but the RFC 3986 unreserved characters, letters, digits and + * "-._~", is encoded, including the ':' in time strings. Servers decode + * before use so over-encoding is harmless, and it is what most encoders + * do. Works like snprintf: the output is always NUL terminated when uOut + * is at least 1, and the return value says how much room was needed. + * + * @param sIn The text to encode. + * @param sOut Receives the encoding. Three times the input length plus + * one always suffices. + * @param uOut The size of sOut. + * @returns The length of the full encoding, not counting the terminator. + * If this is uOut or more the output was cut short. + * @memberof das_url + */ +DAS_API size_t das_url_encode(const char* sIn, char* sOut, size_t uOut); + /** Initialize all fields in an http response to default values. * diff --git a/das3/node.c b/das3/node.c index a2b7bbf..07cd355 100644 --- a/das3/node.c +++ b/das3/node.c @@ -27,8 +27,11 @@ #ifndef _WIN32 #include +#include #else #include +#include +#define getcwd _getcwd #endif #include @@ -1029,6 +1032,65 @@ void del_RootNode(DasNode* pThis) del_NonRootNode(pThis); } +/* ************************************************************************* */ +/* URI spellings, see node_priv.h */ + +static const char* g_aUriShort[] = {"site:", "test:", "local:"}; + +const char* _das_uri_expand(const char* sIn, char* sOut, size_t uOut) +{ + if(uOut == 0) return sOut; + sOut[0] = '\0'; + if(sIn == NULL) return sOut; + for(size_t u = 0; u < sizeof(g_aUriShort)/sizeof(g_aUriShort[0]); ++u){ + size_t uLen = strlen(g_aUriShort[u]); + if(strncmp(sIn, g_aUriShort[u], uLen) != 0) continue; + /* The shorthand alone names the node, "tag:das2.org,2012:site"; + with more after it, it stands for the node plus its ":/" child + separator, as the live catalog composes names top down */ + const char* sRest = sIn + uLen; + while(*sRest == '/') ++sRest; + if(*sRest == '\0') + snprintf(sOut, uOut, D2URI_ROOT "%.*s", (int)(uLen - 1), g_aUriShort[u]); + else + snprintf(sOut, uOut, D2URI_ROOT "%s/%s", g_aUriShort[u], sRest); + return sOut; + } + strncpy(sOut, sIn, uOut - 1); + sOut[uOut - 1] = '\0'; + return sOut; +} + +const char* _das_uri_local(const char* sLoc, char* sOut, size_t uOut) +{ + const char* sTail = sLoc; + const char* sLead = ""; + if(strncmp(sLoc, "file://", 7) == 0) sTail += 7; + else if(strncmp(sLoc, "http://", 7) == 0){ sTail += 7; sLead = "/"; } + else if(strncmp(sLoc, "https://", 8) == 0){ sTail += 8; sLead = "/"; } + + /* A bare relative path: the working directory is part of the name, + * since "./wav.json" means a different file from every other directory. + * One leading "./" is dropped as pure noise. */ + bool bDrive = isalpha((unsigned char)sTail[0]) && (sTail[1] == ':'); + if((sTail == sLoc) && (sTail[0] != '/') && (sTail[0] != '\\') && !bDrive){ + if((sTail[0] == '.') && ((sTail[1] == '/')||(sTail[1] == '\\'))) sTail += 2; + char sCwd[512] = {'\0'}; + if(getcwd(sCwd, sizeof(sCwd) - 1) == NULL) sCwd[0] = '\0'; + snprintf(sOut, uOut, D2URI_LOCAL "%s%s%s", (sCwd[0] == '/') ? "" : "/", sCwd, DAS_DSEPS); + size_t u = strlen(sOut); + if(u < uOut) snprintf(sOut + u, uOut - u, "%s", sTail); + return sOut; + } + snprintf(sOut, uOut, D2URI_LOCAL "%s%s", (bDrive ? "/" : sLead), sTail); + return sOut; +} + +bool _das_uri_isLocal(const char* sUri) +{ + return (sUri != NULL) && (strncmp(sUri, D2URI_LOCAL, sizeof(D2URI_LOCAL) - 1) == 0); +} + /* ************************************************************************* */ /* Root Node Construction */ @@ -1036,8 +1098,13 @@ DasNode* _DasNode_newRootUrl( const char* sUrl, const char* sPathUri, DasCredMngr* pMgr, const char* sAgent, const DasNodeCache* pCache ){ + /* Every node has a URI; one loaded on its own is named by where it is */ + char sUri[600]; + if((sPathUri == NULL)||(sPathUri[0] == '\0')) _das_uri_local(sUrl, sUri, sizeof(sUri)); + else _das_uri_expand(sPathUri, sUri, sizeof(sUri)); + /* Since they want a specific URL, go for the max timeout */ - DasNode* pNode = _DasNode_mkNode(sUrl, sPathUri, pMgr, sAgent, DASHTTP_TO_MAX, pCache); + DasNode* pNode = _DasNode_mkNode(sUrl, sUri, pMgr, sAgent, DASHTTP_TO_MAX, pCache); if(pNode) pNode->bIsRoot = true; return pNode; } @@ -1164,15 +1231,17 @@ DasNode* _DasNode_newRoot( DasNode* pTop = NULL; DasNode* pNode = NULL; - /* "site:/..." and "test:/..." are shorthand for the two well known - children of the das2 root, see D2URI_SITES. Expanding them here, in - the library, means every client gets the same spelling. */ + /* Expanding shorthands here, in the library, means every client gets + the same spelling. Names under local: belong to nodes loaded by + location in some session; the federated roots have never heard of + them and asking would only cost a round trip. */ char sFullUri[600] = {'\0'}; - if((sPathUri != NULL) && ( - (strncmp(sPathUri, "site:", 5) == 0) || (strncmp(sPathUri, "test:", 5) == 0) - )){ - snprintf(sFullUri, sizeof(sFullUri) - 1, "%s%s", D2URI_ROOT, sPathUri); - sPathUri = sFullUri; + if(sPathUri != NULL){ + sPathUri = _das_uri_expand(sPathUri, sFullUri, sizeof(sFullUri)); + if(_das_uri_isLocal(sPathUri)){ + daslog_debug_v("%s is a local: name, those are not routed through the federated catalog", sPathUri); + return NULL; + } } /* Changing node lookup and time outs. New strategy, try to contact a node diff --git a/das3/node.h b/das3/node.h index 8d8b60e..39b99d3 100644 --- a/das3/node.h +++ b/das3/node.h @@ -132,6 +132,14 @@ extern "C" { #define D2URI_ROOT "tag:das2.org,2012:" #define D2URI_SITES D2URI_ROOT "site" #define D2URI_SITE_UIOWA D2URI_SITES ":/uiowa" + +/* Names for nodes that aren't part of the federated catalog system. + * A file or URL loaded with no designated URI gets + * "tag:das2.org,2012:local:" followed by its location. Like a + * private IP range the space is reserved by the machinery, not listed + * anywhere: nothing under it is ever looked up through the federated + * roots. */ +#define D2URI_LOCAL D2URI_ROOT "local:" /* Defines for common document fragments */ #define D2FRAG_TYPE "type" @@ -265,8 +273,12 @@ DAS_API DasNode* new_RootNode( * @param sPathUri Since the node was not loaded by following the federated * catalog system to an end point, you have to tell the node * its name. Any name works, it is saved and reported by - * DasNode_pathUri(), and children get it as a prefix. May - * be the empty string for a root that has no name. + * DasNode_pathUri(), and children get it as a prefix. The + * shorthands "site:", "test:" and "local:" are expanded. + * NULL or empty names the node after its location under + * D2URI_LOCAL, so every node has a URI; give the real + * catalog URI instead when the file stands in for a + * catalog node, see DasCat_mount(). * * @param pMgr A credentials manager object to consult if a password is * requested. May be set to NULL to indicate that only public diff --git a/das3/node_priv.h b/das3/node_priv.h index 2f18fe9..11f4f1e 100644 --- a/das3/node_priv.h +++ b/das3/node_priv.h @@ -84,6 +84,32 @@ struct das_node { const DasNodeCache* pCache; /* borrowed, inherited by children */ }; +/* Catalog URI spellings. + * + * Names are composed top down: the das2 root node has no separator, so + * its child is "tag:das2.org,2012:site", and that node's own separator + * ":/" makes "tag:das2.org,2012:site:/uiowa". The shorthands "site:", + * "test:" and "local:" follow suit. Alone, one names the node: + * "site:" is "tag:das2.org,2012:site". With a path after it, it stands + * for the node and its separator, and a '/' is supplied or collapsed so + * that "site:uiowa" and "site:/uiowa" name one node and stamp one + * requisition. Nothing else is normalized: a full URI is copied as + * written, since separators inside a path belong to the catalog nodes + * and may not be '/'. Returns sOut, empty for a NULL input. */ +const char* _das_uri_expand(const char* sIn, char* sOut, size_t uOut); + +/* The local: name of a node loaded by location rather than found in a + * catalog: D2URI_LOCAL followed by the location. A relative file path is + * prefixed with the working directory, nothing more; symbolic links are + * not resolved, the spelling the user gave is the name. A file:// URL + * loses its scheme, an http(s) URL loses its scheme and gets a '/' before + * the host. Returns sOut. */ +const char* _das_uri_local(const char* sLoc, char* sOut, size_t uOut); + +/* True for URIs under D2URI_LOCAL. These resolve only through a session's + * own mounts, never through the federated roots. */ +bool _das_uri_isLocal(const char* sUri); + /* Root constructors. The public new_RootNode() and new_RootNode_url() in * node.h are thin wrappers that pass NULL for pCache, so trees made through * them are never cached. A DasCat passes its own settings; the session is diff --git a/das3/req.c b/das3/req.c new file mode 100644 index 0000000..312eb51 --- /dev/null +++ b/das3/req.c @@ -0,0 +1,527 @@ +/* Copyright (C) 2026 Chris Piker + * + * This file is part of das2C, the Core Das2 C Library. + * + * Das2C is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License version 2.1 as published + * by the Free Software Foundation. + * + * Das2C is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for + * more details. + * + * You should have received a copy of the GNU Lesser General Public License + * version 2.1 along with das2C; if not, see . + */ + +/* A requisition is strings all the way down, see req.h for why. Nothing + * in this file knows what a key means; that is source.c's job. */ + +#define _POSIX_C_SOURCE 200112L + +#include +#include +#include +#include + +#include "util.h" +#include "log.h" +#include "req_priv.h" + +/* ************************************************************************* */ +/* Helpers */ + +static bool _strieq(const char* a, const char* b) +{ + if((a == NULL)||(b == NULL)) return false; + while(*a && *b){ + if(tolower((unsigned char)*a) != tolower((unsigned char)*b)) return false; + ++a; ++b; + } + return (*a == '\0') && (*b == '\0'); +} + +/* das_strdup() returns NULL for NULL and for "", and here "" is a value: + an empty stamp, an unfilled entry. Every string in a requisition goes + through this so accessors can hand them straight back. */ +static char* _dupz(const char* s) +{ + if((s == NULL)||(s[0] == '\0')) return (char*)calloc(1, 1); + return das_strdup(s); +} + +static bool _setStr(char** ps, const char* s) +{ + char* sNew = _dupz(s); + if(sNew == NULL) return false; + free(*ps); + *ps = sNew; + return true; +} + +bool _das_json_scalar(const DasJdo* p, char* sBuf, size_t uLen) +{ + if((p == NULL)||(uLen == 0)) return false; + switch(p->type){ + case das_json_type_str:{ + const char* s = ((const das_json_str*)p->value)->string; + strncpy(sBuf, s, uLen - 1); + sBuf[uLen - 1] = '\0'; + return true; + } + case das_json_type_num:{ + const das_json_num* pNum = (const das_json_num*)p->value; + size_t u = (pNum->number_size < uLen - 1) ? pNum->number_size : uLen - 1; + memcpy(sBuf, pNum->number, u); + sBuf[u] = '\0'; + return true; + } + case das_json_type_true: strncpy(sBuf, "true", uLen); sBuf[uLen-1] = '\0'; return true; + case das_json_type_false: strncpy(sBuf, "false", uLen); sBuf[uLen-1] = '\0'; return true; + default: return false; + } +} + +/* ************************************************************************* */ +/* Life cycle */ + +DasReq* new_DasReq(void) +{ + DasReq* pThis = (DasReq*)calloc(1, sizeof(DasReq)); + if(pThis == NULL) return NULL; + pThis->sPath = _dupz(NULL); + pThis->sUrl = _dupz(NULL); + pThis->sVersion = _dupz(NULL); + pThis->sLabel = _dupz(NULL); + if(!pThis->sPath || !pThis->sUrl || !pThis->sVersion || !pThis->sLabel){ + del_DasReq(pThis); + return NULL; + } + return pThis; +} + +void del_DasReq(DasReq* pThis) +{ + if(pThis == NULL) return; + DasReq_clear(pThis); + free(pThis->aEntry); + free(pThis->aRaw); + free(pThis->sPath); + free(pThis->sUrl); + free(pThis->sVersion); + free(pThis->sLabel); + free(pThis); +} + +DasReq* copy_DasReq(const DasReq* pThis) +{ + DasReq* pNew = new_DasReq(); + if(pNew == NULL) return NULL; + DasReq_setSrc(pNew, pThis->sPath, pThis->sUrl, pThis->sVersion); + DasReq_setLabel(pNew, pThis->sLabel); + for(size_t u = 0; u < pThis->uEntry; ++u){ + const struct das_req_entry* pE = pThis->aEntry + u; + int n = _DasReq_add(pNew, pE->sKey, pE->sVal, pE->bDerived); + if(n < 0){ del_DasReq(pNew); return NULL; } + _DasReq_setVerdict(pNew, (size_t)n, pE->verdict, pE->sWhy); + } + for(size_t u = 0; u < pThis->uRaw; ++u){ + if(DasReq_setRaw(pNew, pThis->aRaw[u].sName, pThis->aRaw[u].sVal) != DAS_OKAY){ + del_DasReq(pNew); return NULL; + } + } + return pNew; +} + +/* ************************************************************************* */ +/* Stamp and label */ + +void DasReq_setSrc(DasReq* pThis, const char* sPath, const char* sUrl, const char* sVersion) +{ + _setStr(&pThis->sPath, sPath); + _setStr(&pThis->sUrl, sUrl); + _setStr(&pThis->sVersion, sVersion); +} + +const char* DasReq_srcPath(const DasReq* pThis){ return pThis->sPath; } +const char* DasReq_srcUrl(const DasReq* pThis){ return pThis->sUrl; } +const char* DasReq_srcVersion(const DasReq* pThis){ return pThis->sVersion; } +const char* DasReq_label(const DasReq* pThis){ return pThis->sLabel; } +void DasReq_setLabel(DasReq* pThis, const char* sLabel){ _setStr(&pThis->sLabel, sLabel); } + +/* ************************************************************************* */ +/* Entries */ + +static void _freeEntry(struct das_req_entry* pE) +{ + free(pE->sKey); + free(pE->sVal); + free(pE->sWhy); + memset(pE, 0, sizeof(struct das_req_entry)); +} + +int _DasReq_add(DasReq* pThis, const char* sKey, const char* sVal, bool bDerived) +{ + if(pThis->uEntry == pThis->uEntryCap){ + size_t uCap = pThis->uEntryCap ? pThis->uEntryCap * 2 : 8; + struct das_req_entry* a = (struct das_req_entry*)realloc( + pThis->aEntry, uCap * sizeof(struct das_req_entry) + ); + if(a == NULL) return -1; + pThis->aEntry = a; + pThis->uEntryCap = uCap; + } + struct das_req_entry* pE = pThis->aEntry + pThis->uEntry; + memset(pE, 0, sizeof(struct das_req_entry)); + pE->sKey = _dupz(sKey); + pE->sVal = _dupz(sVal); + if((pE->sKey == NULL)||(pE->sVal == NULL)){ _freeEntry(pE); return -1; } + pE->bDerived = bDerived; + pE->nProp = -1; + pThis->uEntry += 1; + return (int)(pThis->uEntry - 1); +} + +void _DasReq_dropDerived(DasReq* pThis) +{ + size_t uKeep = 0; + for(size_t u = 0; u < pThis->uEntry; ++u){ + if(pThis->aEntry[u].bDerived){ _freeEntry(pThis->aEntry + u); continue; } + if(uKeep != u) pThis->aEntry[uKeep] = pThis->aEntry[u]; + ++uKeep; + } + pThis->uEntry = uKeep; +} + +void _DasReq_setVerdict(DasReq* pThis, size_t uIdx, das_req_verdict_e verdict, const char* sWhy) +{ + if(uIdx >= pThis->uEntry) return; + struct das_req_entry* pE = pThis->aEntry + uIdx; + pE->verdict = verdict; + free(pE->sWhy); + pE->sWhy = (verdict == das_req_bad) ? _dupz(sWhy) : NULL; +} + +void _DasReq_clearVerdicts(DasReq* pThis) +{ + for(size_t u = 0; u < pThis->uEntry; ++u){ + pThis->aEntry[u].verdict = das_req_unchecked; + pThis->aEntry[u].nProp = -1; + free(pThis->aEntry[u].sWhy); + pThis->aEntry[u].sWhy = NULL; + } +} + +int DasReq_find(const DasReq* pThis, const char* sKey) +{ + for(size_t u = 0; u < pThis->uEntry; ++u) + if(!pThis->aEntry[u].bDerived && _strieq(pThis->aEntry[u].sKey, sKey)) return (int)u; + return -1; +} + +DasErrCode DasReq_set(DasReq* pThis, const char* sKey, const char* sVal) +{ + if((sKey == NULL)||(sKey[0] == '\0')){ + daslog_error("Requisition keys may not be empty"); + return DASERR_SRC; + } + /* Any change makes the last check stale for the derived entries, they + are the check's output */ + _DasReq_dropDerived(pThis); + + int n = DasReq_find(pThis, sKey); + if(n >= 0){ + struct das_req_entry* pE = pThis->aEntry + n; + char* sNew = _dupz(sVal); + if(sNew == NULL) return DASERR_SRC; + free(pE->sVal); + pE->sVal = sNew; + pE->verdict = das_req_unchecked; + free(pE->sWhy); + pE->sWhy = NULL; + return DAS_OKAY; + } + return (_DasReq_add(pThis, sKey, sVal, false) < 0) ? DASERR_SRC : DAS_OKAY; +} + +const char* DasReq_value(const DasReq* pThis, const char* sKey) +{ + int n = DasReq_find(pThis, sKey); + return (n < 0) ? NULL : pThis->aEntry[n].sVal; +} + +bool DasReq_remove(DasReq* pThis, const char* sKey) +{ + int n = DasReq_find(pThis, sKey); + if(n < 0) return false; + _freeEntry(pThis->aEntry + n); + memmove(pThis->aEntry + n, pThis->aEntry + n + 1, + (pThis->uEntry - n - 1) * sizeof(struct das_req_entry)); + pThis->uEntry -= 1; + _DasReq_dropDerived(pThis); + return true; +} + +void DasReq_clear(DasReq* pThis) +{ + for(size_t u = 0; u < pThis->uEntry; ++u) _freeEntry(pThis->aEntry + u); + pThis->uEntry = 0; + for(size_t u = 0; u < pThis->uRaw; ++u){ + free(pThis->aRaw[u].sName); + free(pThis->aRaw[u].sVal); + } + pThis->uRaw = 0; +} + +size_t DasReq_count(const DasReq* pThis){ return pThis->uEntry; } + +const char* DasReq_key(const DasReq* pThis, size_t u) +{ + return (u < pThis->uEntry) ? pThis->aEntry[u].sKey : NULL; +} + +const char* DasReq_val(const DasReq* pThis, size_t u) +{ + return (u < pThis->uEntry) ? pThis->aEntry[u].sVal : NULL; +} + +das_req_verdict_e DasReq_verdict(const DasReq* pThis, size_t u) +{ + return (u < pThis->uEntry) ? pThis->aEntry[u].verdict : das_req_unchecked; +} + +const char* DasReq_why(const DasReq* pThis, size_t u) +{ + if((u >= pThis->uEntry)||(pThis->aEntry[u].sWhy == NULL)) return ""; + return pThis->aEntry[u].sWhy; +} + +bool DasReq_derived(const DasReq* pThis, size_t u) +{ + return (u < pThis->uEntry) ? pThis->aEntry[u].bDerived : false; +} + +/* ************************************************************************* */ +/* Raw parameters. Names are protocol names and compare exactly. */ + +DasErrCode DasReq_setRaw(DasReq* pThis, const char* sName, const char* sVal) +{ + if((sName == NULL)||(sName[0] == '\0')){ + daslog_error("Protocol parameter names may not be empty"); + return DASERR_SRC; + } + for(size_t u = 0; u < pThis->uRaw; ++u){ + if(strcmp(pThis->aRaw[u].sName, sName) == 0) + return _setStr(&pThis->aRaw[u].sVal, sVal) ? DAS_OKAY : DASERR_SRC; + } + if(pThis->uRaw == pThis->uRawCap){ + size_t uCap = pThis->uRawCap ? pThis->uRawCap * 2 : 4; + struct das_req_raw* a = (struct das_req_raw*)realloc( + pThis->aRaw, uCap * sizeof(struct das_req_raw) + ); + if(a == NULL) return DASERR_SRC; + pThis->aRaw = a; + pThis->uRawCap = uCap; + } + struct das_req_raw* pR = pThis->aRaw + pThis->uRaw; + pR->sName = _dupz(sName); + pR->sVal = _dupz(sVal); + if((pR->sName == NULL)||(pR->sVal == NULL)){ + free(pR->sName); free(pR->sVal); + return DASERR_SRC; + } + pThis->uRaw += 1; + return DAS_OKAY; +} + +size_t DasReq_rawCount(const DasReq* pThis){ return pThis->uRaw; } + +const char* DasReq_raw(const DasReq* pThis, size_t u, const char** psVal) +{ + if(u >= pThis->uRaw) return NULL; + if(psVal) *psVal = pThis->aRaw[u].sVal; + return pThis->aRaw[u].sName; +} + +/* ************************************************************************* */ +/* Saving. A hand rolled writer because the document is three dicts of + * strings and das_json has no builder. */ + +typedef struct { char* s; size_t u; size_t uCap; bool bFail; } _out_t; + +static void _put(_out_t* pO, const char* s) +{ + size_t n = strlen(s); + if(pO->u + n + 1 > pO->uCap){ + size_t uCap = (pO->uCap ? pO->uCap * 2 : 256); + while(uCap < pO->u + n + 1) uCap *= 2; + char* sNew = (char*)realloc(pO->s, uCap); + if(sNew == NULL){ pO->bFail = true; return; } + pO->s = sNew; + pO->uCap = uCap; + } + memcpy(pO->s + pO->u, s, n + 1); + pO->u += n; +} + +static void _putStr(_out_t* pO, const char* s) +{ + _put(pO, "\""); + char sEsc[8]; + for(const unsigned char* p = (const unsigned char*)s; *p; ++p){ + switch(*p){ + case '"': _put(pO, "\\\""); break; + case '\\': _put(pO, "\\\\"); break; + case '\n': _put(pO, "\\n"); break; + case '\r': _put(pO, "\\r"); break; + case '\t': _put(pO, "\\t"); break; + default: + if(*p < 0x20){ snprintf(sEsc, sizeof(sEsc), "\\u%04x", *p); _put(pO, sEsc); } + else{ sEsc[0] = (char)*p; sEsc[1] = '\0'; _put(pO, sEsc); } + } + } + _put(pO, "\""); +} + +static void _putPair(_out_t* pO, const char* sKey, const char* sVal, bool bFirst) +{ + if(!bFirst) _put(pO, ",\n"); + _put(pO, " "); + _putStr(pO, sKey); + _put(pO, ": "); + _putStr(pO, sVal); +} + +char* DasReq_text(const DasReq* pThis) +{ + _out_t o = {NULL, 0, 0, false}; + _put(&o, "{\n \"source\": {\n"); + _putPair(&o, "path", pThis->sPath, true); + _putPair(&o, "url", pThis->sUrl, false); + _putPair(&o, "version", pThis->sVersion, false); + _put(&o, "\n },\n \"label\": "); + _putStr(&o, pThis->sLabel); + _put(&o, ",\n \"settings\": {\n"); + bool bFirst = true; + for(size_t u = 0; u < pThis->uEntry; ++u){ + if(pThis->aEntry[u].bDerived) continue; + _putPair(&o, pThis->aEntry[u].sKey, pThis->aEntry[u].sVal, bFirst); + bFirst = false; + } + _put(&o, "\n },\n \"params\": {\n"); + for(size_t u = 0; u < pThis->uRaw; ++u) + _putPair(&o, pThis->aRaw[u].sName, pThis->aRaw[u].sVal, u == 0); + _put(&o, "\n }\n}\n"); + if(o.bFail){ free(o.s); return NULL; } + return o.s; +} + +DasErrCode DasReq_write(const DasReq* pThis, const char* sPath) +{ + char* sText = DasReq_text(pThis); + if(sText == NULL){ + daslog_error("Couldn't allocate the requisition text"); + return DASERR_SRC; + } + FILE* pOut = fopen(sPath, "wb"); + if(pOut == NULL){ + daslog_error_v("Couldn't open %s for writing", sPath); + free(sText); + return DASERR_SRC; + } + bool bOk = (fputs(sText, pOut) >= 0); + fclose(pOut); + free(sText); + if(!bOk){ + daslog_error_v("Couldn't write %s", sPath); + return DASERR_SRC; + } + return DAS_OKAY; +} + +/* Example settings in nodes are written as JSON paths, + "coords/time/props/min". Saved requisitions may use either that or + the dotted key, so a slash path is folded to a key on the way in: + drop "props", join with dots. Section aliases are left for the + source's matcher, which accepts them anyway. */ +static void _pathToKey(const char* sPath, char* sKey, size_t uLen) +{ + size_t u = 0; + const char* pBeg = sPath; + sKey[0] = '\0'; + while(*pBeg && (u + 2 < uLen)){ + const char* pEnd = strchr(pBeg, '/'); + size_t n = pEnd ? (size_t)(pEnd - pBeg) : strlen(pBeg); + if((n != 5)||(strncmp(pBeg, "props", 5) != 0)){ + if(n > uLen - u - 2) n = uLen - u - 2; + if(u > 0) sKey[u++] = '.'; + memcpy(sKey + u, pBeg, n); + u += n; + sKey[u] = '\0'; + } + if(pEnd == NULL) break; + pBeg = pEnd + 1; + } +} + +static const char* _jstr(const DasJdo* p, const char* sPath) +{ + const DasJdo* pSub = p ? DasJdo_get(p, sPath) : NULL; + return ((pSub != NULL)&&(pSub->type == das_json_type_str)) ? DasJdo_string(pSub) : NULL; +} + +DasReq* new_DasReq_text(const char* sJson, size_t uLen) +{ + DasJdo* pDoc = das_json_parse(sJson, uLen); + if((pDoc == NULL)||(pDoc->type != das_json_type_dict)){ + daslog_error("Requisition text is not a JSON object"); + free(pDoc); + return NULL; + } + DasReq* pThis = new_DasReq(); + if(pThis == NULL){ free(pDoc); return NULL; } + + const DasJdo* pSrc = DasJdo_get(pDoc, "source"); + DasReq_setSrc(pThis, _jstr(pSrc, "path"), _jstr(pSrc, "url"), _jstr(pSrc, "version")); + DasReq_setLabel(pThis, _jstr(pDoc, "label")); + + char sKey[256], sVal[1024]; + bool bOk = true; + const DasJdo* pSet = DasJdo_get(pDoc, "settings"); + for(const das_json_dict_el* pEl = DasJdo_dictFirst(pSet); bOk && pEl; pEl = pEl->next){ + if(strchr(pEl->name->string, '/')) _pathToKey(pEl->name->string, sKey, sizeof(sKey)); + else{ strncpy(sKey, pEl->name->string, sizeof(sKey) - 1); sKey[sizeof(sKey)-1] = '\0'; } + if(!_das_json_scalar(pEl->value, sVal, sizeof(sVal))) sVal[0] = '\0'; + bOk = (DasReq_set(pThis, sKey, sVal) == DAS_OKAY); + } + const DasJdo* pRaw = DasJdo_get(pDoc, "params"); + for(const das_json_dict_el* pEl = DasJdo_dictFirst(pRaw); bOk && pEl; pEl = pEl->next){ + if(!_das_json_scalar(pEl->value, sVal, sizeof(sVal))) sVal[0] = '\0'; + bOk = (DasReq_setRaw(pThis, pEl->name->string, sVal) == DAS_OKAY); + } + free(pDoc); + if(!bOk){ del_DasReq(pThis); return NULL; } + return pThis; +} + +DasReq* new_DasReq_file(const char* sPath) +{ + FILE* pIn = fopen(sPath, "rb"); + if(pIn == NULL){ + daslog_error_v("Couldn't open requisition file %s", sPath); + return NULL; + } + fseek(pIn, 0, SEEK_END); + long nLen = ftell(pIn); + fseek(pIn, 0, SEEK_SET); + if(nLen < 0){ fclose(pIn); daslog_error_v("Couldn't size %s", sPath); return NULL; } + char* sBuf = (char*)malloc((size_t)nLen + 1); + if(sBuf == NULL){ fclose(pIn); return NULL; } + size_t uRead = fread(sBuf, 1, (size_t)nLen, pIn); + fclose(pIn); + sBuf[uRead] = '\0'; + DasReq* pThis = new_DasReq_text(sBuf, uRead); + free(sBuf); + return pThis; +} diff --git a/das3/req.h b/das3/req.h new file mode 100644 index 0000000..641c922 --- /dev/null +++ b/das3/req.h @@ -0,0 +1,268 @@ +/* Copyright (C) 2026 Chris Piker + * + * This file is part of das2C, the Core Das2 C Library. + * + * Das2C is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License version 2.1 as published + * by the Free Software Foundation. + * + * Das2C is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for + * more details. + * + * You should have received a copy of the GNU Lesser General Public License + * version 2.1 along with das2C; if not, see . + */ + +/** @file req.h Requisitions: what a client wants from a data source */ + +#ifndef _das_req_h_ +#define _das_req_h_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** @addtogroup catalog + * @{ + */ + +/** @defgroup req Requisitions + * + * A requisition is the list of settings a client wants from a data + * source. It is the form an application fills in, and it is deliberately + * simple. A DasReq holds strings naming entries in the interface section + * of a data source node and nothing else: no hidden pointers, no parsed + * values and no protocol parameters. Everything that interprets a + * requisition is a function of the DasSrc it is handed to, see + * DasSrc_check() and DasStreamSrc_url() in source.h. + * + * That split lets a requisition outlive the interface it was built from. + * It can be saved to a file one day, reloaded the next, and used to query + * the data source again. + * + * @par Entries and verdicts + * + * Each entry is a key, a value and a verdict. Keys are the dotted names + * described under @ref src_keys. Shorter spellings are not interpreted + * by the requisition; they are kept as entered and the DasSrc resolves + * them. Verdicts start out unchecked. DasSrc_check() validates every + * entry against a source, marking each one ok or bad and attaching a + * reason to the bad ones. + * + * Sources create requisitions. DasSrc_newReq() hands back one seeded + * with a worked example, or a blank one listing every required key with + * an empty value so that a form knows what must be filled. Either way + * the requisition is stamped with the source URI, and a later + * DasSrc_check() refuses to validate it against a different source. A + * requisition made with new_DasReq() has no stamp and is validated by + * whatever source it is handed to. + * + * @par Raw parameters + * + * A requisition may also carry raw protocol parameters: name and value + * pairs sent to the server exactly as given. These bypass the interface + * entirely and are for people who already know the server. The source + * applies them after everything else, so they win. This is not the + * intended use of a requisition, only an escape hatch. + * + * @par Saved requisitions + * + * DasReq_write() and new_DasReq_file() round trip a requisition through a + * small JSON document, the same shape as an example in a source node with + * a `source` block in front: + * + * @code + * { "source": { "path": "tag:das2.org,2012:site:/uiowa/juno/wav/survey", + * "url": "https://jupiter.physics.uiowa.edu/.../flex.json", + * "version": "0.7" }, + * "label": "Perijove 16, 30 second bins", + * "settings": { "coords/time/props/min": "2018-10-02T11:00", + * "coords/time/props/max": "2018-10-04T11:00", + * "coords/time/props/res": "30" }, + * "params": { "extra": "1" } } + * @endcode + * + * The path is the source's identity and is what a later check compares; + * the URL records where that copy of the node came from. The version is + * the node format version. + * + * @todo Issue #42 tracks a content version. Until the catalog spec adds + * one, a stale requisition shows up as unknown keys at check time. + */ + +/** How an entry fared the last time a source looked at it */ +typedef enum das_req_verdict_enum { + das_req_unchecked = 0, /**< not validated since it was set */ + das_req_ok, /**< the key resolved and the value is well formed */ + das_req_bad /**< see DasReq_why() */ +} das_req_verdict_e; + +/** A requisition. Create with new_DasReq() or DasSrc_newReq(), free with + * del_DasReq(). Not thread safe for writers. */ +typedef struct das_req DasReq; + +/** An empty requisition with no source stamp */ +DAS_API DasReq* new_DasReq(void); + +/** A requisition read from the JSON text written by DasReq_text(). + * @returns a new requisition, or NULL after logging what was wrong with + * the text. */ +DAS_API DasReq* new_DasReq_text(const char* sJson, size_t uLen); + +/** A requisition read from a file written by DasReq_write(), or NULL + * after logging why not. */ +DAS_API DasReq* new_DasReq_file(const char* sPath); + +/** A deep copy, verdicts included */ +DAS_API DasReq* copy_DasReq(const DasReq* pThis); + +DAS_API void del_DasReq(DasReq* pThis); + +/* ************************************************************************* */ +/* The stamp and the label */ + +/** Record which source this requisition is for. + * + * DasSrc_newReq() does this for you. Call it yourself when adopting a + * requisition built from loose strings so that later checks hold it to + * one source. + * + * @param sPath The source's catalog URI, see DasSrc_path(). "" removes + * the stamp. + * @param sUrl Where the source document was loaded from, kept for + * diagnostics; it plays no part in matching. + * @param sVersion The node format version, or "". + * @memberof DasReq + */ +DAS_API void DasReq_setSrc( + DasReq* pThis, const char* sPath, const char* sUrl, const char* sVersion +); + +/** The catalog URI of the stamped source, "" when unstamped. + * @memberof DasReq */ +DAS_API const char* DasReq_srcPath(const DasReq* pThis); + +/** The URL of the stamped source, "" when unstamped. @memberof DasReq */ +DAS_API const char* DasReq_srcUrl(const DasReq* pThis); + +/** The node format version of the stamped source, "" when unknown. + * @memberof DasReq */ +DAS_API const char* DasReq_srcVersion(const DasReq* pThis); + +/** A short description, the example label when seeded from one, else + * "". @memberof DasReq */ +DAS_API const char* DasReq_label(const DasReq* pThis); + +/** @memberof DasReq */ +DAS_API void DasReq_setLabel(DasReq* pThis, const char* sLabel); + +/* ************************************************************************* */ +/* Entries */ + +/** Set a value. + * + * The key and value are stored as given. Nothing is resolved or checked + * here, there is no source to ask; the entry's verdict becomes unchecked + * and the next DasSrc_check() validates it. Setting a key that is already + * present replaces its value. Keys compare case insensitively. + * + * @param sKey Any spelling from @ref src_keys, or a coordinate name + * alone for the range forms described there. + * @param sVal The value. "" is allowed and means the key is wanted but + * not yet filled in, which is how a blank requisition lists what + * a source requires. NULL is treated as "". + * @returns DAS_OKAY, or DASERR_SRC if memory ran out or the key is empty. + * @memberof DasReq + */ +DAS_API DasErrCode DasReq_set(DasReq* pThis, const char* sKey, const char* sVal); + +/** The value stored under a key, exactly as set, or NULL when the key is + * not present. Only keys set directly are found, the derived entries a + * check adds are reached by index. @memberof DasReq */ +DAS_API const char* DasReq_value(const DasReq* pThis, const char* sKey); + +/** Drop an entry. @returns true if there was one. @memberof DasReq */ +DAS_API bool DasReq_remove(DasReq* pThis, const char* sKey); + +/** Drop every entry and raw parameter, keeping the stamp and label. + * @memberof DasReq */ +DAS_API void DasReq_clear(DasReq* pThis); + +/** The number of entries, including any a check derived, see + * DasReq_derived(). @memberof DasReq */ +DAS_API size_t DasReq_count(const DasReq* pThis); + +/** Entry number uIdx's key, or NULL past the end. @memberof DasReq */ +DAS_API const char* DasReq_key(const DasReq* pThis, size_t uIdx); + +/** Entry number uIdx's value, or NULL past the end. @memberof DasReq */ +DAS_API const char* DasReq_val(const DasReq* pThis, size_t uIdx); + +/** @memberof DasReq */ +DAS_API das_req_verdict_e DasReq_verdict(const DasReq* pThis, size_t uIdx); + +/** Why entry number uIdx was rejected, "" unless its verdict is + * das_req_bad. The text names the key and is meant to be shown as is. + * @memberof DasReq */ +DAS_API const char* DasReq_why(const DasReq* pThis, size_t uIdx); + +/** True for entries a check added rather than the client. + * + * A coordinate range given as one key, `time=2026-01-02,2026-01-03`, + * expands to entries for the minimum and maximum, and a required key the + * client never set is added with an empty value and a bad verdict so a + * form can mark it. Derived entries are rebuilt on every check and + * vanish on the next set; a panel usually skips them when drawing and a + * client reading values back should too. + * @memberof DasReq + */ +DAS_API bool DasReq_derived(const DasReq* pThis, size_t uIdx); + +/** Index of the entry for a key, or -1. @memberof DasReq */ +DAS_API int DasReq_find(const DasReq* pThis, const char* sKey); + +/* ************************************************************************* */ +/* Raw parameters */ + +/** Name a protocol parameter directly, see the group description. + * @returns DAS_OKAY, or DASERR_SRC if memory ran out. @memberof DasReq */ +DAS_API DasErrCode DasReq_setRaw(DasReq* pThis, const char* sName, const char* sVal); + +/** @memberof DasReq */ +DAS_API size_t DasReq_rawCount(const DasReq* pThis); + +/** Raw parameter number uIdx. + * @param psVal Receives the value, may be NULL. + * @returns The name, or NULL past the end. @memberof DasReq */ +DAS_API const char* DasReq_raw(const DasReq* pThis, size_t uIdx, const char** psVal); + +/* ************************************************************************* */ +/* Saving */ + +/** The requisition as JSON text, see the group description. + * + * Derived entries and verdicts are not written; they belong to a check, + * not to the requisition. + * + * @returns A heap string to be released with free(), or NULL if memory + * ran out. + * @memberof DasReq + */ +DAS_API char* DasReq_text(const DasReq* pThis); + +/** Write DasReq_text() to a file, replacing any existing file. + * @returns DAS_OKAY, or DASERR_SRC after logging why not. @memberof DasReq */ +DAS_API DasErrCode DasReq_write(const DasReq* pThis, const char* sPath); + +/** @} */ +/** @} */ + +#ifdef __cplusplus +} +#endif + +#endif /* _das_req_h_ */ diff --git a/das3/req_priv.h b/das3/req_priv.h new file mode 100644 index 0000000..d5ce652 --- /dev/null +++ b/das3/req_priv.h @@ -0,0 +1,87 @@ +/* Copyright (C) 2026 Chris Piker + * + * This file is part of das2C, the Core Das2 C Library. + * + * Das2C is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License version 2.1 as published + * by the Free Software Foundation. + * + * Das2C is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for + * more details. + * + * You should have received a copy of the GNU Lesser General Public License + * version 2.1 along with das2C; if not, see . + */ + +/* Requisition internals shared by req.c and source.c. Not installed. + * + * source.c is the only writer of verdicts and derived entries, so those + * operations are here rather than in req.h: an application has no + * business validating a requisition. + */ + +#ifndef _das_req_priv_h_ +#define _das_req_priv_h_ + +#include "req.h" +#include "json.h" + +struct das_req_entry { + char* sKey; /* owned, as the client typed it */ + char* sVal; /* owned, never NULL, may be "" */ + das_req_verdict_e verdict; + char* sWhy; /* owned, NULL unless bad */ + bool bDerived; + int nProp; /* index of the source property the last check + resolved this entry to, -1 if none. Only + meaningful to the source that set it. */ +}; + +struct das_req_raw { + char* sName; + char* sVal; +}; + +struct das_req { + char* sPath; /* stamp, each "" when unknown */ + char* sUrl; + char* sVersion; + char* sLabel; + + struct das_req_entry* aEntry; + size_t uEntry; + size_t uEntryCap; + + struct das_req_raw* aRaw; + size_t uRaw; + size_t uRawCap; +}; + +/* Append an entry without looking for a duplicate. Used for derived + * entries, which are keyed by full key and rebuilt wholesale, so the + * uniqueness DasReq_set() maintains for client keys is not needed. + * Returns the new index, or -1 if memory ran out. */ +int _DasReq_add(DasReq* pThis, const char* sKey, const char* sVal, bool bDerived); + +/* Drop every derived entry. Called at the start of a check so that a + * second check never sees its own output. */ +void _DasReq_dropDerived(DasReq* pThis); + +/* Set the verdict of an entry, with a reason for a bad one (ignored + * otherwise). sWhy is copied. */ +void _DasReq_setVerdict(DasReq* pThis, size_t uIdx, das_req_verdict_e verdict, const char* sWhy); + +/* Mark every entry unchecked. DasReq_set() does this for the one entry + * it touches; a check that is about to validate everything again calls this. */ +void _DasReq_clearVerdicts(DasReq* pThis); + +/* Copy a JSON scalar as text: strings as they are, numbers in their + * source notation, true and false as those words. Returns false for + * null, containers and missing values, leaving sBuf untouched. The one + * reader of JSON number text in the catalog code, since das_json keeps + * numbers as unterminated text and a default of 86.4 should stay 86.4. */ +bool _das_json_scalar(const DasJdo* pJdo, char* sBuf, size_t uLen); + +#endif /* _das_req_priv_h_ */ diff --git a/das3/sett_priv.h b/das3/sett_priv.h deleted file mode 100644 index 861e924..0000000 --- a/das3/sett_priv.h +++ /dev/null @@ -1,145 +0,0 @@ -/* Copyright (C) 2026 Chris Piker - * - * This file is part of das2C, the Core Das2 C Library. - * - * Das2C is free software; you can redistribute it and/or modify it under - * the terms of the GNU Lesser General Public License version 2.1 as published - * by the Free Software Foundation. - * - * Das2C is distributed in the hope that it will be useful, but WITHOUT ANY - * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for - * more details. - * - * You should have received a copy of the GNU Lesser General Public License - * version 2.1 along with das2C; if not, see . - */ - -/* Settable internals shared by settable.c and source.c. Not installed. - * - * The struct is here rather than in settable.c because source.c builds the - * list when a source node loads and walks it when matching keys, and C has - * no package-private. Applications see only the accessors in settable.h. - */ - -#ifndef _das_sett_priv_h_ -#define _das_sett_priv_h_ - -#include "settable.h" -#include "json.h" - -/* One settable. Borrowed pointers point into the node's parsed document, - * which the node keeps for its whole life, so nothing here is copied that - * the document already holds in a usable form. JSON number text is not - * NUL terminated, hence the owned copies of default, enum and range - * strings. - * - * Roughly a dozen pointers per settable. The hsk node, the largest in the - * wild, has 73 variables and comes to about 150 settables, so a list is a - * few tens of kilobytes rather than the 700 KB the fixed arrays of the - * first fetcher cost. - */ -struct das_settable { - das_src_sec_e sec; - das_settable_type_e type; /* resolved at parse, see _DasSettable_type() */ - - const char* sGroup; /* borrowed dict key, "" for options */ - const char* sProp; /* borrowed dict key, the node's spelling */ - char* sKey; /* owned, "coords.time.min" */ - - const DasJdo* pGroup; /* the coordinate, variable or format dict, - NULL for options */ - const DasJdo* pProp; /* the property dict: value, set, type, label */ - const DasJdo* pSet; /* the set dict: param, flag, pval, enum, range */ - const DasJdo* pParam; /* the protocol parameter entry for sParam, or - NULL if the node does not describe it */ - const char* sParam; /* borrowed from pSet, never NULL, see below */ - const char* sTypeName; /* borrowed, "" when inferred */ - bool bRequired; /* set/required or the parameter's required */ - - char* sDefVal; /* owned, NULL when the node's value is null */ - char** aEnum; /* owned, enum values in node order */ - size_t uEnum; - char* sLo; /* owned, range ends, NULL when none stated */ - char* sHi; -}; - -/* Build the settable list for a source node. - * - * Walks the four interface sections in das_src_sec_e order and makes one - * entry for every property that has a "set" dict, so entries come out - * grouped by section and then in document order, which is the order a - * settings panel should show them in. Both interface layouts in the wild - * are read: the 2018 one with "coordinates", properties beside the group's - * own label and units, and long property names; and the current one with - * "coords" and properties under "props". The layout is not reported, the - * entries look the same either way. - * - * A property whose set block has no "param" cannot be sent anywhere. Such - * an entry is dropped with a warning naming the key instead of failing the - * load, so a node with one bad entry still describes and serves the rest. - * Nothing else in the interface can fail: unknown types become text and - * missing titles are composed on demand. - * - * @param pIface The node's "interface" dict. NULL or a non-dict gives an - * empty list, which is not an error; a source may have nothing to - * set. - * - * @param pParams The protocol's parameter dict, "httpParams" or - * "http_params", used for the type, required flag and range of a - * settable that does not state its own. May be NULL. - * - * @param puCount Receives the number of entries. - * - * @returns A heap array of exactly *puCount entries, to be released with - * _DasSettable_freeList(), or NULL only when memory could not be - * had, in which case *puCount is 0 and an error was logged. An - * empty list is a non-NULL pointer to nothing in particular, so - * callers test the count, not the pointer. - */ -DasSettable* _DasSettable_parse( - const DasJdo* pIface, const DasJdo* pParams, size_t* puCount -); - -/* Release a list from _DasSettable_parse() and the strings it owns. The - * document the entries pointed into is not touched. */ -void _DasSettable_freeList(DasSettable* aList, size_t uCount); - -/* Key grammar helpers, see the "Keys" block in settable.h. Queries match - * user input with these; keeping them next to the parser means the - * spellings accepted on input and the spellings written into keys cannot - * drift apart. */ - -/* Fold a section name as a user may type it, "coord", "coords", - * "coordinate", "fmt", "opts" and so on, to a section. Case insensitive. - * Returns false if the word is not a section name. */ -bool _das_src_secByName(const char* sName, das_src_sec_e* pSec); - -/* The short spelling of a property name: "minimum" -> "min", "enable" -> - * "enabled". Names with no alias are returned as given. Used on both - * sides of a comparison so that either spelling matches either. */ -const char* _das_src_propName(const char* sProp); - -/* True if two property names are the same after _das_src_propName() and - * case folding. */ -bool _das_src_propEq(const char* sA, const char* sB); - -/* Split a dotted key into at most three parts. - * - * @param sKey The key as typed. - * @param aParts Receives the parts, each NUL terminated. DAS_MAX_ID_BUFSZ - * per part is enough for anything a node may contain. - * @returns The number of parts, 1 to 3, or 0 if the key is empty, has an - * empty part, has more than three parts, or has a part that does - * not fit. - */ -int _das_src_splitKey(const char* sKey, char aParts[3][DAS_MAX_ID_BUFSZ]); - -/* Copy a JSON scalar as text: strings as they are, numbers in their source - * notation, true and false as those words. Returns false for null, - * containers and missing values, leaving sBuf untouched. This is the one - * reader of JSON number text in the catalog code, since das_json keeps - * numbers as unterminated text and a default of 86.4 should stay 86.4. */ -bool _das_json_scalar(const DasJdo* pJdo, char* sBuf, size_t uLen); - -#endif /* _das_sett_priv_h_ */ diff --git a/das3/settable.c b/das3/settable.c deleted file mode 100644 index 08e8698..0000000 --- a/das3/settable.c +++ /dev/null @@ -1,537 +0,0 @@ -/* Copyright (C) 2026 Chris Piker - * - * This file is part of das2C, the Core Das2 C Library. - * - * Das2C is free software; you can redistribute it and/or modify it under - * the terms of the GNU Lesser General Public License version 2.1 as published - * by the Free Software Foundation. - * - * Das2C is distributed in the hope that it will be useful, but WITHOUT ANY - * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for - * more details. - * - * You should have received a copy of the GNU Lesser General Public License - * version 2.1 along with das2C; if not, see . - */ - -#define _POSIX_C_SOURCE 200112L - -#include -#include -#include -#include - -#include "util.h" -#include "log.h" -#include "time.h" -#include "sett_priv.h" - -/* Section names as the current catalogs spell them. Keys are built from - * these; input accepts the singular forms too, see _das_src_secByName(). */ -static const char* g_sSecName[das_src_sec_size] = {"coords", "data", "formats", "options"}; - -/* ************************************************************************* */ -/* Small string and JSON helpers. DasJdo_get() takes slash paths, so the - * dots in parameter names such as "read.time.min" are not special to it. */ - -static bool _strieq(const char* a, const char* b) -{ - if((a == NULL)||(b == NULL)) return false; - while(*a && *b){ - if(tolower((unsigned char)*a) != tolower((unsigned char)*b)) return false; - ++a; ++b; - } - return (*a == '\0') && (*b == '\0'); -} - -static void _strcpyz(char* sDest, const char* sSrc, size_t uDest) -{ - if(uDest == 0) return; - strncpy(sDest, sSrc ? sSrc : "", uDest - 1); - sDest[uDest - 1] = '\0'; -} - -static const DasJdo* _jget(const DasJdo* p, const char* sPath) -{ - return ((p == NULL)||(sPath == NULL)) ? NULL : DasJdo_get(p, sPath); -} - -static const char* _jstr(const DasJdo* p, const char* sPath) -{ - const DasJdo* pSub = _jget(p, sPath); - return ((pSub != NULL)&&(pSub->type == das_json_type_str)) ? DasJdo_string(pSub) : NULL; -} - -static bool _jisdict(const DasJdo* p){ return (p != NULL)&&(p->type == das_json_type_dict); } -static bool _jisary(const DasJdo* p){ return (p != NULL)&&(p->type == das_json_type_ary); } -static bool _jtrue(const DasJdo* p){ return (p != NULL)&&(p->type == das_json_type_true); } -static bool _jisbool(const DasJdo* p){ - return (p != NULL)&&((p->type == das_json_type_true)||(p->type == das_json_type_false)); -} - -bool _das_json_scalar(const DasJdo* p, char* sBuf, size_t uLen) -{ - if((p == NULL)||(uLen == 0)) return false; - switch(p->type){ - case das_json_type_str: - _strcpyz(sBuf, ((const das_json_str*)p->value)->string, uLen); - return true; - case das_json_type_num:{ - const das_json_num* pNum = (const das_json_num*)p->value; - size_t u = (pNum->number_size < uLen - 1) ? pNum->number_size : uLen - 1; - memcpy(sBuf, pNum->number, u); - sBuf[u] = '\0'; - return true; - } - case das_json_type_true: _strcpyz(sBuf, "true", uLen); return true; - case das_json_type_false: _strcpyz(sBuf, "false", uLen); return true; - default: return false; - } -} - -/* A heap copy of a scalar, NULL for anything that is not one. Defaults, - * enum entries and range ends are short in practice; the longest seen is a - * few dozen bytes, so anything past this is cut rather than carried. */ -#define SCALAR_SZ 1024 -static char* _scalarDup(const DasJdo* p) -{ - char sBuf[SCALAR_SZ]; - if(!_das_json_scalar(p, sBuf, sizeof(sBuf))) return NULL; - return das_strdup(sBuf); -} - -static bool _parseBool(const char* s, bool* pOut) -{ - if(s == NULL) return false; - if(_strieq(s,"true")||_strieq(s,"yes")||_strieq(s,"on")||_strieq(s,"1")|| - _strieq(s,"t")||_strieq(s,"y")){ *pOut = true; return true; } - if(_strieq(s,"false")||_strieq(s,"no")||_strieq(s,"off")||_strieq(s,"0")|| - _strieq(s,"f")||_strieq(s,"n")){ *pOut = false; return true; } - return false; -} - -/* ************************************************************************* */ -/* Key grammar */ - -const char* das_src_sec_name(das_src_sec_e sec) -{ - return ((sec >= 0)&&(sec < das_src_sec_size)) ? g_sSecName[sec] : ""; -} - -bool _das_src_secByName(const char* s, das_src_sec_e* pSec) -{ - if(_strieq(s,"coord")||_strieq(s,"coords")||_strieq(s,"coordinate")||_strieq(s,"coordinates")){ - *pSec = das_src_sec_coord; return true; - } - if(_strieq(s,"data")){ *pSec = das_src_sec_data; return true; } - if(_strieq(s,"format")||_strieq(s,"formats")||_strieq(s,"fmt")){ - *pSec = das_src_sec_format; return true; - } - if(_strieq(s,"option")||_strieq(s,"options")||_strieq(s,"opt")||_strieq(s,"opts")){ - *pSec = das_src_sec_option; return true; - } - return false; -} - -const char* _das_src_propName(const char* s) -{ - if(s == NULL) return ""; - if(_strieq(s, "minimum")||_strieq(s, "min")) return "min"; - if(_strieq(s, "maximum")||_strieq(s, "max")) return "max"; - if(_strieq(s, "resolution")||_strieq(s, "res")) return "res"; - if(_strieq(s, "interval")||_strieq(s, "inter")) return "inter"; - if(_strieq(s, "enable")||_strieq(s, "enabled")) return "enabled"; - return s; -} - -bool _das_src_propEq(const char* a, const char* b) -{ - return _strieq(_das_src_propName(a), _das_src_propName(b)); -} - -int _das_src_splitKey(const char* sKey, char aParts[3][DAS_MAX_ID_BUFSZ]) -{ - if(sKey == NULL) return 0; - int n = 0; - const char* pBeg = sKey; - while(true){ - const char* pEnd = strchr(pBeg, '.'); - size_t uLen = pEnd ? (size_t)(pEnd - pBeg) : strlen(pBeg); - if((uLen == 0)||(uLen >= DAS_MAX_ID_BUFSZ)||(n == 3)) return 0; - memcpy(aParts[n], pBeg, uLen); - aParts[n][uLen] = '\0'; - ++n; - if(pEnd == NULL) break; - pBeg = pEnd + 1; - } - return n; -} - -/* ************************************************************************* */ -/* Parsing */ - -/* Fold a type word from a node to the enum. NULL and unknown words are - * text, the caller decides whether that is final. */ -static bool _typeByName(const char* s, das_settable_type_e* pType) -{ - if(s == NULL) return false; - if(_strieq(s,"boolean")||_strieq(s,"bool")){ *pType = das_settable_bool; return true; } - if(_strieq(s,"integer")||_strieq(s,"int")){ *pType = das_settable_int; return true; } - if(_strieq(s,"real")||_strieq(s,"double")||_strieq(s,"float")){ - *pType = das_settable_real; return true; - } - if(_strieq(s,"isotime")||_strieq(s,"time")||_strieq(s,"datetime")){ - *pType = das_settable_time; return true; - } - if(_strieq(s,"enum")){ *pType = das_settable_enum; return true; } - if(_strieq(s,"string")||_strieq(s,"text")){ *pType = das_settable_text; return true; } - return false; -} - -/* Preference order: a set enum list, the property's own type, a boolean - * default, a flag toggle, the protocol parameter's type, then text. The - * enum list goes first because a property declared "string" with a list of - * choices is still a list of choices to anyone presenting it. A flag set - * parameter type is a server side packing detail and says nothing about - * the single toggle that feeds it. */ -static void _resolveType(DasSettable* p) -{ - p->sTypeName = _jstr(p->pProp, "type"); - if(p->sTypeName == NULL) p->sTypeName = ""; - - if(p->uEnum > 0){ p->type = das_settable_enum; return; } - if(_typeByName(p->sTypeName, &(p->type))){ - if(p->type != das_settable_enum) return; - } - if(_jisbool(_jget(p->pProp, "value")) || (_jstr(p->pSet, "flag") != NULL)){ - p->type = das_settable_bool; return; - } - const char* sParamType = _jstr(p->pParam, "type"); - if(_typeByName(sParamType, &(p->type)) && (p->type != das_settable_enum)){ - if(p->sTypeName[0] == '\0') p->sTypeName = sParamType; - return; - } - p->type = das_settable_text; -} - -/* Enum entries are usually dicts carrying value plus per item pval or flag - * overrides, but a bare scalar list is accepted too. */ -static bool _fillEnum(DasSettable* p) -{ - const DasJdo* pEnum = _jget(p->pSet, "enum"); - if(!_jisary(pEnum)) return true; - - size_t uMax = ((const das_json_ary*)pEnum->value)->length; - if(uMax == 0) return true; - p->aEnum = (char**)calloc(uMax, sizeof(char*)); - if(p->aEnum == NULL) return false; - - for(const das_json_ary_el* pEl = DasJdo_aryFirst(pEnum); pEl; pEl = pEl->next){ - const DasJdo* pVal = _jisdict(pEl->value) ? _jget(pEl->value, "value") : pEl->value; - char* s = _scalarDup(pVal); - if(s == NULL) continue; - p->aEnum[p->uEnum] = s; - p->uEnum += 1; - } - return true; -} - -static void _fillRange(DasSettable* p) -{ - const DasJdo* pRange = _jget(p->pSet, "range"); - if(!_jisary(pRange)) pRange = _jget(p->pParam, "range"); - if(!_jisary(pRange) && (p->sec == das_src_sec_coord) && - (_das_src_propEq(p->sProp, "min") || _das_src_propEq(p->sProp, "max"))) - pRange = _jget(p->pGroup, "validRange"); - if(!_jisary(pRange)) return; - - const das_json_ary_el* pEl = DasJdo_aryFirst(pRange); - if(pEl == NULL) return; - p->sLo = _scalarDup(pEl->value); - if(pEl->next) p->sHi = _scalarDup(pEl->next->value); -} - -/* One property. With aOut NULL only the count is wanted. Returns false - * on allocation failure, the only way filling can fail. */ -static bool _addProp( - das_src_sec_e sec, const char* sGroup, const DasJdo* pGroup, - const char* sProp, const DasJdo* pProp, const DasJdo* pParams, - DasSettable* aOut, size_t* puCount -){ - if(!_jisdict(pProp)) return true; - const DasJdo* pSet = _jget(pProp, "set"); - if(!_jisdict(pSet)) return true; /* documentation only, not settable */ - - const char* sParam = _jstr(pSet, "param"); - if(sParam == NULL){ - if(aOut != NULL) - daslog_warn_v("Ignoring %s.%s%s%s, its set block names no parameter", - g_sSecName[sec], sGroup, sGroup[0] ? "." : "", sProp); - return true; - } - - if(aOut == NULL){ *puCount += 1; return true; } - - DasSettable* p = aOut + *puCount; - memset(p, 0, sizeof(DasSettable)); - p->sec = sec; - p->sGroup = sGroup; - p->sProp = sProp; - p->pGroup = pGroup; - p->pProp = pProp; - p->pSet = pSet; - p->sParam = sParam; - p->pParam = _jget(pParams, sParam); - p->bRequired = _jtrue(_jget(pSet, "required")) || _jtrue(_jget(p->pParam, "required")); - - size_t uKey = strlen(g_sSecName[sec]) + strlen(sGroup) + strlen(sProp) + 3; - p->sKey = (char*)malloc(uKey); - if(p->sKey == NULL) return false; - if(sGroup[0] == '\0') - snprintf(p->sKey, uKey, "%s.%s", g_sSecName[sec], sProp); - else - snprintf(p->sKey, uKey, "%s.%s.%s", g_sSecName[sec], sGroup, sProp); - - p->sDefVal = _scalarDup(_jget(pProp, "value")); - if(!_fillEnum(p)) return false; - _fillRange(p); - _resolveType(p); - - *puCount += 1; - return true; -} - -/* A group is a coordinate, a data variable, a format, or the whole options - * block. Current groups keep properties under "props", 2018 groups hold - * them directly beside label/title/units. Anything without a 'set' is - * skipped either way, so iterating the group itself is safe for the old - * layout. */ -static bool _addGroup( - das_src_sec_e sec, const char* sGroup, const DasJdo* pGroup, - const DasJdo* pParams, DasSettable* aOut, size_t* puCount -){ - if(!_jisdict(pGroup)) return true; - const DasJdo* pProps = _jget(pGroup, "props"); - const DasJdo* pIter = _jisdict(pProps) ? pProps : pGroup; - for(const das_json_dict_el* pEl = DasJdo_dictFirst(pIter); pEl; pEl = pEl->next){ - if(!_addProp(sec, sGroup, pGroup, pEl->name->string, pEl->value, pParams, aOut, puCount)) - return false; - } - return true; -} - -static bool _addSection( - das_src_sec_e sec, const DasJdo* pSec, const DasJdo* pParams, - DasSettable* aOut, size_t* puCount -){ - if(!_jisdict(pSec)) return true; - for(const das_json_dict_el* pEl = DasJdo_dictFirst(pSec); pEl; pEl = pEl->next){ - if(!_addGroup(sec, pEl->name->string, pEl->value, pParams, aOut, puCount)) - return false; - } - return true; -} - -DasSettable* _DasSettable_parse(const DasJdo* pIface, const DasJdo* pParams, size_t* puCount) -{ - *puCount = 0; - if(!_jisdict(pIface)) pIface = NULL; - - /* Two passes over the same walk: count, allocate exactly that many, - fill. Options are a group with no enclosing section, so they go - through _addGroup directly and their pGroup is NULL. */ - size_t uCount = 0; - const DasJdo* pSec = _jget(pIface, "coords"); - if(pSec == NULL) pSec = _jget(pIface, "coordinates"); - const DasJdo* pOpts = _jget(pIface, "options"); - - _addSection(das_src_sec_coord, pSec, pParams, NULL, &uCount); - _addSection(das_src_sec_data, _jget(pIface, "data"), pParams, NULL, &uCount); - _addSection(das_src_sec_format, _jget(pIface, "formats"), pParams, NULL, &uCount); - _addGroup(das_src_sec_option, "", pOpts, pParams, NULL, &uCount); - - /* An empty list is still a list: one byte keeps malloc from handing - back NULL on some platforms and lets callers test the count alone. */ - DasSettable* aOut = (DasSettable*)calloc(uCount ? uCount : 1, sizeof(DasSettable)); - if(aOut == NULL){ - daslog_error_v("Couldn't allocate %zu source settings", uCount); - return NULL; - } - - size_t uFill = 0; - bool bOk = - _addSection(das_src_sec_coord, pSec, pParams, aOut, &uFill) && - _addSection(das_src_sec_data, _jget(pIface, "data"), pParams, aOut, &uFill) && - _addSection(das_src_sec_format, _jget(pIface, "formats"), pParams, aOut, &uFill) && - _addGroup(das_src_sec_option, "", pOpts, pParams, aOut, &uFill); - - if(!bOk){ - daslog_error_v("Couldn't allocate the strings of source setting %zu", uFill); - _DasSettable_freeList(aOut, uFill + 1); - return NULL; - } - *puCount = uFill; - return aOut; -} - -void _DasSettable_freeList(DasSettable* aList, size_t uCount) -{ - if(aList == NULL) return; - for(size_t u = 0; u < uCount; ++u){ - DasSettable* p = aList + u; - free(p->sKey); - free(p->sDefVal); - for(size_t v = 0; v < p->uEnum; ++v) free(p->aEnum[v]); - free(p->aEnum); - free(p->sLo); - free(p->sHi); - } - free(aList); -} - -/* ************************************************************************* */ -/* Accessors */ - -das_src_sec_e DasSettable_section(const DasSettable* p){ return p->sec; } -const char* DasSettable_group(const DasSettable* p){ return p->sGroup; } -const char* DasSettable_prop(const DasSettable* p){ return p->sProp; } -const char* DasSettable_key(const DasSettable* p){ return p->sKey; } -const char* DasSettable_param(const DasSettable* p){ return p->sParam; } -bool DasSettable_required(const DasSettable* p){ return p->bRequired; } -const char* DasSettable_defVal(const DasSettable* p){ return p->sDefVal; } -das_settable_type_e DasSettable_type(const DasSettable* p){ return p->type; } -const char* DasSettable_typeName(const DasSettable* p){ return p->sTypeName; } -size_t DasSettable_enumCount(const DasSettable* p){ return p->uEnum; } - -const char* DasSettable_enum(const DasSettable* p, size_t uIdx) -{ - return (uIdx < p->uEnum) ? p->aEnum[uIdx] : NULL; -} - -bool DasSettable_range(const DasSettable* p, const char** psLo, const char** psHi) -{ - if((p->sLo == NULL)&&(p->sHi == NULL)) return false; - if(psLo) *psLo = p->sLo ? p->sLo : ""; - if(psHi) *psHi = p->sHi ? p->sHi : ""; - return true; -} - -/* Units for display. Current nodes put a units string on the property; - * both layouts may carry a units dict with a "value" at the group level, - * under props in the current layout and directly in the old one. */ -const char* DasSettable_units(const DasSettable* p) -{ - const DasJdo* pU = _jget(p->pProp, "units"); - if(pU != NULL){ - if(pU->type == das_json_type_str) return DasJdo_string(pU); - if(_jstr(pU, "value")) return _jstr(pU, "value"); - } - if(p->sec == das_src_sec_coord){ - const char* s = _jstr(p->pGroup, "props/units/value"); - if(s == NULL) s = _jstr(p->pGroup, "units/value"); - if(s != NULL) return s; - } - if(p->type == das_settable_time) return "UTC"; - return ""; -} - -/* What to call a group in composed text: its label, name or title, falling - * back to the dict key */ -static const char* _groupLabel(const DasSettable* p) -{ - const char* s = _jstr(p->pGroup, "label"); - if(s == NULL) s = _jstr(p->pGroup, "name"); - if(s == NULL) s = _jstr(p->pGroup, "title"); - return s ? s : p->sGroup; -} - -const char* DasSettable_title(const DasSettable* p, char* sBuf, size_t uLen) -{ - const char* s = _jstr(p->pProp, "title"); - if(s == NULL) s = _jstr(p->pProp, "summary"); - if(s == NULL) s = _jstr(p->pProp, "label"); - if(s == NULL) s = _jstr(p->pProp, "name"); - if(s != NULL) return s; - if(p->sec == das_src_sec_option) return ""; - - const char* sProp = _das_src_propName(p->sProp); - if(strcmp(sProp, "enabled") == 0){ - if(p->sec == das_src_sec_format){ - snprintf(sBuf, uLen, "Output as %s", _groupLabel(p)); - } - else{ - /* A variable's title is usually the physical quantity, which - reads better after "Output" than its short label does */ - const char* sTitle = _jstr(p->pGroup, "title"); - snprintf(sBuf, uLen, "Output %s", sTitle ? sTitle : _groupLabel(p)); - } - return sBuf; - } - if(p->sec == das_src_sec_coord){ - const char* sFmt = NULL; - if(strcmp(sProp, "min") == 0) sFmt = "Minimum %s value to output"; - else if(strcmp(sProp, "max") == 0) sFmt = "Maximum %s value to output"; - else if(strcmp(sProp, "res") == 0) sFmt = "Average over %s bins of this width"; - else if(strcmp(sProp, "inter") == 0) sFmt = "Step between output %s values"; - if(sFmt != NULL){ - snprintf(sBuf, uLen, sFmt, _groupLabel(p)); - return sBuf; - } - } - snprintf(sBuf, uLen, "%s %s", _groupLabel(p), p->sProp); - return sBuf; -} - -/* ************************************************************************* */ -/* Validation. Failures are logged and returned, not raised through - * das_error(), because these are user typed values and the default error - * disposition would exit the application over a typo. */ - -DasErrCode DasSettable_check(const DasSettable* p, const char* sVal) -{ - if(sVal == NULL) sVal = ""; - char* pEnd = NULL; - - switch(p->type){ - case das_settable_bool:{ - bool b; - if(_parseBool(sVal, &b)) return DAS_OKAY; - daslog_error_v("'%s' is not a boolean value for %s, use true or false", sVal, p->sKey); - return DASERR_SRC; - } - case das_settable_enum:{ - for(size_t u = 0; u < p->uEnum; ++u) - if(_strieq(p->aEnum[u], sVal)) return DAS_OKAY; - - char sChoices[512] = {'\0'}; - size_t u = 0; - for(size_t v = 0; (v < p->uEnum) && (u < sizeof(sChoices)); ++v) - u += snprintf(sChoices + u, sizeof(sChoices) - u, "%s%s", v ? ", " : "", p->aEnum[v]); - daslog_error_v("'%s' is not an allowed value for %s, choose one of: %s", - sVal, p->sKey, sChoices); - return DASERR_SRC; - } - case das_settable_int: - strtol(sVal, &pEnd, 10); - if((pEnd != sVal) && (*pEnd == '\0')) return DAS_OKAY; - daslog_error_v("'%s' is not an integer value for %s", sVal, p->sKey); - return DASERR_SRC; - case das_settable_real: - strtod(sVal, &pEnd); - if((pEnd != sVal) && (*pEnd == '\0')) return DAS_OKAY; - daslog_error_v("'%s' is not a real value for %s", sVal, p->sKey); - return DASERR_SRC; - case das_settable_time:{ - /* "now" is how nodes spell an open ended range and servers take - it in requests too */ - das_time dt; - if(_strieq(sVal, "now") || dt_parsetime(&dt, sVal)) return DAS_OKAY; - daslog_error_v("'%s' is not a time value for %s, use ISO 8601 such as " - "2026-01-15T12:00", sVal, p->sKey); - return DASERR_SRC; - } - default: - return DAS_OKAY; - } -} diff --git a/das3/settable.h b/das3/settable.h deleted file mode 100644 index 5951dab..0000000 --- a/das3/settable.h +++ /dev/null @@ -1,233 +0,0 @@ -/* Copyright (C) 2026 Chris Piker - * - * This file is part of das2C, the Core Das2 C Library. - * - * Das2C is free software; you can redistribute it and/or modify it under - * the terms of the GNU Lesser General Public License version 2.1 as published - * by the Free Software Foundation. - * - * Das2C is distributed in the hope that it will be useful, but WITHOUT ANY - * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for - * more details. - * - * You should have received a copy of the GNU Lesser General Public License - * version 2.1 along with das2C; if not, see . - */ - -/** @file settable.h The adjustable properties of a data source */ - -#ifndef _das_settable_h_ -#define _das_settable_h_ - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/** @addtogroup catalog - * @{ - */ - -/** @defgroup settable Source settings - * - * A data source node describes what a client may ask for in its "interface" - * section: coordinate ranges, which data variables to include, output - * formats and server specific options. Each entry that a client may change - * carries a "set" block saying how a value for it reaches the server. - * The library flattens those entries into a list of DasSettable objects - * when a source node loads, one per changeable property, and the node - * hands them out by index. Everything else in the interface is - * documentation and is reached through DasNode_getJdo() as before. - * - * A DasSettable is a description. It says what a property is, what values - * it takes and what it is called, and it never changes after the node - * loads. Values chosen for a particular request live in a query object - * (see source.h), so one node can serve any number of queries at once. - * - * @anchor settable_keys - * ### Keys - * - * Every settable has a dotted key built from its section, group and - * property names as the node spells them, for example `coords.time.min`, - * `data.lshell.enabled`, `formats.csv.delim` or `options.text`. Options - * have no group, so their keys have two parts. Keys are how a request - * names a settable, and a query accepts the following shorter spellings as - * well, so long as the result is unambiguous within the source: - * - * - Section names in singular or plural form, `coord` or `coords`, - * `format` or `formats`, `option` or `options`, and `fmt` or `opt`. - * - The group and property alone, `time.min`, when only one section has - * that group. - * - A group alone, `lshell` or `data.lshell`, meaning the group's - * `enabled` toggle. - * - An option alone, `text`, when no group has that name. - * - The property names `min`, `max`, `res` and `inter` may be written - * out as `minimum`, `maximum`, `resolution` and `interval`, and - * `enable` as `enabled`. Nodes from 2018 spell them the long way, - * later nodes the short way, and users may type either. - * - * Matching is case insensitive. A spelling that fits more than one - * settable is an error that names the candidates, not a guess. Section - * names are ordered as in das_src_sec_e, which is also the order nodes - * present them in. - */ - -/** The parts of a source interface, in presentation order. - * - * `das_src_sec_size` is the number of real sections and is handy for - * sizing per section arrays. - */ -typedef enum das_src_sec_enum { - das_src_sec_coord = 0, /**< "coords": time, frequency, L-shell ... */ - das_src_sec_data, /**< "data": the variables a source can emit */ - das_src_sec_format, /**< "formats": output serializations and their knobs */ - das_src_sec_option, /**< "options": server specific switches, no groups */ - das_src_sec_size -} das_src_sec_e; - -/** The name of a section as it appears in keys, "coords", "data", - * "formats" or "options". Returns "" for das_src_sec_size. */ -DAS_API const char* das_src_sec_name(das_src_sec_e sec); - -/** The kind of value a settable takes. - * - * Source nodes name types with a handful of spellings that have drifted - * over the years. They are folded into this set when the node loads so - * that clients need not know the history. A type the library does not - * recognize is reported as das_settable_text and the node's spelling of it - * is still available via DasSettable_typeName(). - */ -typedef enum das_settable_type_enum { - das_settable_text = 0, /**< any string, including unrecognized types */ - das_settable_bool, /**< true or false, with the spellings DasSettable_check() allows */ - das_settable_int, /**< an integer in decimal */ - das_settable_real, /**< a real number */ - das_settable_time, /**< a UTC time, ISO 8601 preferred */ - das_settable_enum /**< one of a fixed list, see DasSettable_enumCount() */ -} das_settable_type_e; - -/** One changeable property of a data source. - * - * Settables are owned by the source node that lists them and are valid for - * as long as that node is. Applications never create or free them; they - * ask the node for them by index. All accessors take a const pointer and - * may be called from several threads at once. - */ -typedef struct das_settable DasSettable; - -/** The section of the interface this settable came from */ -DAS_API das_src_sec_e DasSettable_section(const DasSettable* pThis); - -/** The group name as the node spells it. - * - * For a coordinate that is the coordinate name, `time`; for a variable or - * format its name, `lshell` or `csv`. Options have no group and return - * "" (never NULL), so the result is always safe to print. - */ -DAS_API const char* DasSettable_group(const DasSettable* pThis); - -/** The property name as the node spells it, `min` or `minimum`, - * `enabled`, `delim`. Use DasSettable_key() rather than rebuilding keys - * from this, since the key is what a query matches against. */ -DAS_API const char* DasSettable_prop(const DasSettable* pThis); - -/** The full dotted key, see @ref settable_keys. Unique within a source. */ -DAS_API const char* DasSettable_key(const DasSettable* pThis); - -/** A short human readable title for the property. - * - * The node's own title is returned when it has one. Coordinate range - * properties and enable toggles usually have none, since their meaning is - * conventional, so a title is composed for them in sBuf from the group's - * label, for example "Minimum Time value to output" or "Output as CSV". - * Anything else without a title gets the group label and property name. - * - * @param sBuf Space for a composed title. 64 bytes covers the composed - * forms, longer group labels are cut to fit. - * @param uLen The size of sBuf. - * @returns A pointer to the title, which is either inside the node's - * document or is sBuf. Never NULL, but may be "" for an option - * with no title at all. - */ -DAS_API const char* DasSettable_title(const DasSettable* pThis, char* sBuf, size_t uLen); - -/** The units a value is expressed in, as a string the node supplies, or - * "UTC" for time typed settables with no stated units, or "" when there - * are none. Units are for display and are not checked against values. */ -DAS_API const char* DasSettable_units(const DasSettable* pThis); - -/** The kind of value this settable takes */ -DAS_API das_settable_type_e DasSettable_type(const DasSettable* pThis); - -/** The node's own spelling of the type, "isotime", "real", "boolean" and - * so on, or "" when the node gave none and the type was inferred. For - * display, DasSettable_type() is the one to branch on. */ -DAS_API const char* DasSettable_typeName(const DasSettable* pThis); - -/** True when a query must supply a value, or the server will refuse the - * request. A required settable with a default is satisfied by the default, - * see DasSettable_defVal(). */ -DAS_API bool DasSettable_required(const DasSettable* pThis); - -/** The value in effect when a query sets nothing, as the node spells it, - * or NULL if the node has no default. Numbers are given in the node's - * own notation, "86.4", not reformatted. */ -DAS_API const char* DasSettable_defVal(const DasSettable* pThis); - -/** The name of the protocol parameter that carries this setting to the - * server, "start_time" or "read.time.min". Several settables may share a - * parameter, boolean toggles that pack into one flag set in particular. - * Mostly of interest when composing requests by hand. */ -DAS_API const char* DasSettable_param(const DasSettable* pThis); - -/** The number of allowed values for an enumerated settable, 0 for any - * other type. */ -DAS_API size_t DasSettable_enumCount(const DasSettable* pThis); - -/** Allowed value number uIdx of an enumerated settable, as the node spells - * it, or NULL when uIdx is out of range. DasSettable_check() accepts - * these in any letter case; the value a query sends is the node's. */ -DAS_API const char* DasSettable_enum(const DasSettable* pThis, size_t uIdx); - -/** The stated range of a numeric or time settable. - * - * Nodes put ranges in several places: on the set block, on the protocol - * parameter, or for coordinate `min` and `max` on the coordinate itself as - * its valid range. All three are consulted, in that order. The upper end - * of a time range is often the word "now". - * - * @param psLo Receives the low end, as the node spells it. May be NULL. - * @param psHi Receives the high end. May be NULL. - * @returns true if a range was stated, false otherwise, in which case the - * output pointers are not touched. - */ -DAS_API bool DasSettable_range( - const DasSettable* pThis, const char** psLo, const char** psHi -); - -/** Decide if a string is an acceptable value for this settable. - * - * Booleans accept true, false, yes, no, on, off, 1 and 0 in any letter - * case. Enumerations accept any listed value, case insensitively. - * Integers, reals and times are checked for form only, not against the - * stated range, because servers are the authority on what they hold and - * ranges in nodes are routinely out of date. Text is always acceptable. - * - * Queries call this before storing a value, so applications rarely need to, - * but a settings panel may want to validate as the user types. - * - * @returns DAS_OKAY, or DASERR_SRC after logging a message that names the - * key, the offending value and, for an enumeration, the choices. - */ -DAS_API DasErrCode DasSettable_check(const DasSettable* pThis, const char* sVal); - -/** @} */ -/** @} */ - -#ifdef __cplusplus -} -#endif - -#endif /* _das_settable_h_ */ diff --git a/das3/source.c b/das3/source.c new file mode 100644 index 0000000..99fae6d --- /dev/null +++ b/das3/source.c @@ -0,0 +1,1727 @@ +/* Copyright (C) 2026 Chris Piker + * + * This file is part of das2C, the Core Das2 C Library. + * + * Das2C is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License version 2.1 as published + * by the Free Software Foundation. + * + * Das2C is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for + * more details. + * + * You should have received a copy of the GNU Lesser General Public License + * version 2.1 along with das2C; if not, see . + */ + +/* Data sources, see source.h. + * + * Failures over content, a broken node or a bad user value, are logged + * and returned, never raised through das_error(), because this code + * meets bad catalogs and typos every day and the default error + * disposition would exit the program over either. Clients that catch + * log messages see everything; clients that do not still get verdicts. + * + * Grew out of the pilot in utilities/das_fetch.c, which carried the + * translation rules for two node layouts. Those rules are not written + * down anywhere else yet; until the catalog spec states them, the + * comments on _translate() are the reference. + */ + +#define _POSIX_C_SOURCE 200112L + +#include +#include +#include +#include +#include + +#include "util.h" +#include "log.h" +#include "time.h" +#include "http.h" +#include "node.h" +#include "node_priv.h" +#include "src_priv.h" + +/* Section names as the current catalogs spell them. Keys are built from + * these; input accepts the singular forms too, see _das_src_secByName(). */ +static const char* g_sSecName[das_src_sec_size] = {"coords", "data", "formats", "options"}; + +/* Scalars copied out of the document are short in practice, the longest + * seen is a few dozen bytes; anything past this is cut rather than + * carried. */ +#define SCALAR_SZ 1024 + +/* ************************************************************************* */ +/* Small string and JSON helpers. DasJdo_get() takes slash paths, so the + * dots in parameter names such as "read.time.min" are not special to it. */ + +static bool _strieq(const char* a, const char* b) +{ + if((a == NULL)||(b == NULL)) return false; + while(*a && *b){ + if(tolower((unsigned char)*a) != tolower((unsigned char)*b)) return false; + ++a; ++b; + } + return (*a == '\0') && (*b == '\0'); +} + +static void _strcpyz(char* sDest, const char* sSrc, size_t uDest) +{ + if(uDest == 0) return; + strncpy(sDest, sSrc ? sSrc : "", uDest - 1); + sDest[uDest - 1] = '\0'; +} + +static const DasJdo* _jget(const DasJdo* p, const char* sPath) +{ + return ((p == NULL)||(sPath == NULL)) ? NULL : DasJdo_get(p, sPath); +} + +static const char* _jstr(const DasJdo* p, const char* sPath) +{ + const DasJdo* pSub = _jget(p, sPath); + return ((pSub != NULL)&&(pSub->type == das_json_type_str)) ? DasJdo_string(pSub) : NULL; +} + +static bool _jisdict(const DasJdo* p){ return (p != NULL)&&(p->type == das_json_type_dict); } +static bool _jisary(const DasJdo* p){ return (p != NULL)&&(p->type == das_json_type_ary); } +static bool _jtrue(const DasJdo* p){ return (p != NULL)&&(p->type == das_json_type_true); } +static bool _jisbool(const DasJdo* p){ + return (p != NULL)&&((p->type == das_json_type_true)||(p->type == das_json_type_false)); +} +static size_t _jlen(const DasJdo* p) +{ + if(_jisdict(p)) return ((const das_json_dict*)p->value)->length; + if(_jisary(p)) return ((const das_json_ary*)p->value)->length; + return 0; +} + +/* das_strdup() returns NULL for "", which here would turn an empty default + into no default. Empty strings are copied as empty strings. */ +static char* _strdupz(const char* s) +{ + if((s == NULL)||(s[0] == '\0')) return (char*)calloc(1, 1); + return das_strdup(s); +} + +/* Every string a source keeps, keys, default values, enum items and range + * ends, is its own small allocation, freed with the source. A possible + * optimization, not taken: the source owns pDoc for its whole life, so + * plain JSON strings could be referenced in place and only numbers, which + * need formatting, copied. That would remove most of these allocations + * at the cost of a copied-or-borrowed flag per string. A full string + * arena was considered and rejected: it needs an exact size pass that + * duplicates the formatting logic below, and it hides overruns from + * valgrind. Revisit if a heap profile of a browser build points here. */ +static char* _scalarDup(const DasJdo* p) +{ + char sBuf[SCALAR_SZ]; + if(!_das_json_scalar(p, sBuf, sizeof(sBuf))) return NULL; + return _strdupz(sBuf); +} + +static bool _parseBool(const char* s, bool* pOut) +{ + if(s == NULL) return false; + if(_strieq(s,"true")||_strieq(s,"yes")||_strieq(s,"on")||_strieq(s,"1")|| + _strieq(s,"t")||_strieq(s,"y")){ *pOut = true; return true; } + if(_strieq(s,"false")||_strieq(s,"no")||_strieq(s,"off")||_strieq(s,"0")|| + _strieq(s,"f")||_strieq(s,"n")){ *pOut = false; return true; } + return false; +} + +/* snprintf that appends and never runs the cursor past the buffer */ +static size_t _catf(char* sBuf, size_t uLen, size_t u, const char* sFmt, ...) +{ + if(u >= uLen) return u; + va_list ap; + va_start(ap, sFmt); + int n = vsnprintf(sBuf + u, uLen - u, sFmt, ap); + va_end(ap); + if(n < 0) return u; + return (u + (size_t)n < uLen) ? u + (size_t)n : uLen; +} + +/* ************************************************************************* */ +/* Key grammar */ + +const char* das_src_sec_name(das_src_sec_e sec) +{ + return ((sec >= 0)&&(sec < das_src_sec_size)) ? g_sSecName[sec] : ""; +} + +bool _das_src_secByName(const char* s, das_src_sec_e* pSec) +{ + if(_strieq(s,"coord")||_strieq(s,"coords")||_strieq(s,"coordinate")||_strieq(s,"coordinates")){ + *pSec = das_src_sec_coord; return true; + } + if(_strieq(s,"data")){ *pSec = das_src_sec_data; return true; } + if(_strieq(s,"format")||_strieq(s,"formats")||_strieq(s,"fmt")){ + *pSec = das_src_sec_format; return true; + } + if(_strieq(s,"option")||_strieq(s,"options")||_strieq(s,"opt")||_strieq(s,"opts")){ + *pSec = das_src_sec_option; return true; + } + return false; +} + +const char* _das_src_propName(const char* s) +{ + if(s == NULL) return ""; + if(_strieq(s, "minimum")||_strieq(s, "min")) return "min"; + if(_strieq(s, "maximum")||_strieq(s, "max")) return "max"; + if(_strieq(s, "resolution")||_strieq(s, "res")) return "res"; + if(_strieq(s, "interval")||_strieq(s, "inter")) return "inter"; + if(_strieq(s, "enable")||_strieq(s, "enabled")) return "enabled"; + return s; +} + +bool _das_src_propEq(const char* a, const char* b) +{ + return _strieq(_das_src_propName(a), _das_src_propName(b)); +} + +int _das_src_splitKey(const char* sKey, char aParts[3][DAS_MAX_ID_BUFSZ]) +{ + if(sKey == NULL) return 0; + int n = 0; + const char* pBeg = sKey; + while(true){ + const char* pEnd = strchr(pBeg, '.'); + size_t uLen = pEnd ? (size_t)(pEnd - pBeg) : strlen(pBeg); + if((uLen == 0)||(uLen >= DAS_MAX_ID_BUFSZ)||(n == 3)) return 0; + memcpy(aParts[n], pBeg, uLen); + aParts[n][uLen] = '\0'; + ++n; + if(pEnd == NULL) break; + pBeg = pEnd + 1; + } + return n; +} + +/* ************************************************************************* */ +/* Parsing the interface */ + +/* Fold a type word from a node to the enum. NULL and unknown words are + * text, the caller decides whether that is final. */ +static bool _typeByName(const char* s, das_srcprop_type_e* pType) +{ + if(s == NULL) return false; + if(_strieq(s,"boolean")||_strieq(s,"bool")){ *pType = das_srcprop_bool; return true; } + if(_strieq(s,"integer")||_strieq(s,"int")){ *pType = das_srcprop_int; return true; } + if(_strieq(s,"real")||_strieq(s,"double")||_strieq(s,"float")){ + *pType = das_srcprop_real; return true; + } + if(_strieq(s,"isotime")||_strieq(s,"time")||_strieq(s,"datetime")){ + *pType = das_srcprop_time; return true; + } + if(_strieq(s,"enum")){ *pType = das_srcprop_enum; return true; } + if(_strieq(s,"string")||_strieq(s,"text")){ *pType = das_srcprop_text; return true; } + return false; +} + +/* Preference order: a set enum list, the property's own type, a boolean + * default, a flag toggle, the protocol parameter's type, the shape of the + * value, then text. The enum list goes first because a property declared + * "string" with a list of choices is still a list of choices to anyone + * presenting it. A flag set parameter type is a server side packing + * detail and says nothing about the single toggle that feeds it. */ +static void _resolveType(DasSrcProp* p) +{ + p->sTypeName = _jstr(p->pJdo, "type"); + if(p->sTypeName == NULL) p->sTypeName = ""; + const DasSrcCtrl* pC = p->pCtrl; + const DasJdo* pVal = _jget(p->pJdo, "value"); + + if(pC && (pC->uEnum > 0)){ p->type = das_srcprop_enum; return; } + if(_typeByName(p->sTypeName, &(p->type))){ + if(p->type != das_srcprop_enum) return; + } + if(_jisbool(pVal) || (pC && (_jstr(pC->pSet, "flag") != NULL))){ + p->type = das_srcprop_bool; return; + } + const char* sParamType = pC ? _jstr(pC->pParam, "type") : NULL; + if(_typeByName(sParamType, &(p->type)) && (p->type != das_srcprop_enum)){ + if(p->sTypeName[0] == '\0') p->sTypeName = sParamType; + return; + } + if((pVal != NULL)&&(pVal->type == das_json_type_num)){ p->type = das_srcprop_real; return; } + p->type = das_srcprop_text; +} + +/* Enum entries are usually dicts carrying value plus per item pval or flag + * overrides, but a bare scalar list is accepted too. */ +static bool _fillEnum(DasSrcCtrl* p) +{ + const DasJdo* pEnum = _jget(p->pSet, "enum"); + if(!_jisary(pEnum)) return true; + + size_t uMax = _jlen(pEnum); + if(uMax == 0) return true; + p->aEnum = (char**)calloc(uMax, sizeof(char*)); + if(p->aEnum == NULL) return false; + + for(const das_json_ary_el* pEl = DasJdo_aryFirst(pEnum); pEl; pEl = pEl->next){ + const DasJdo* pVal = _jisdict(pEl->value) ? _jget(pEl->value, "value") : pEl->value; + char* s = _scalarDup(pVal); + if(s == NULL) continue; + p->aEnum[p->uEnum] = s; + p->uEnum += 1; + } + return true; +} + +static void _fillRange(DasSrcCtrl* p) +{ + const DasSrcProp* pProp = p->pProp; + const DasJdo* pRange = _jget(p->pSet, "range"); + if(!_jisary(pRange)) pRange = _jget(p->pParam, "range"); + if(!_jisary(pRange) && (pProp->sec == das_src_sec_coord) && + (_das_src_propEq(pProp->sName, "min") || _das_src_propEq(pProp->sName, "max"))) + pRange = _jget(pProp->pGroup->pJdo, "validRange"); + if(!_jisary(pRange)) return; + + const das_json_ary_el* pEl = DasJdo_aryFirst(pRange); + if(pEl == NULL) return; + p->sLo = _scalarDup(pEl->value); + if(pEl->next) p->sHi = _scalarDup(pEl->next->value); +} + +/* Release what a control owns; the control itself lives in aCtrl */ +static void _clearCtrl(DasSrcCtrl* p) +{ + if(p == NULL) return; + for(size_t v = 0; v < p->uEnum; ++v) free(p->aEnum[v]); + free(p->aEnum); + free(p->sLo); + free(p->sHi); +} + +/* A member of a group dict is a property if it is a dict that looks like + * one. In the current layout every member of "props" qualifies; in the + * 2018 layout properties sit beside the group's own name and title + * strings, and a units dict with a value counts as the units property + * just as it does under props today. */ +static bool _isPropDict(const DasJdo* p) +{ + if(!_jisdict(p)) return false; + return (_jget(p, "value") != NULL)||(_jget(p, "set") != NULL)||(_jget(p, "type") != NULL); +} + +static const DasJdo* _groupProps(const DasJdo* pGroup) +{ + const DasJdo* pProps = _jget(pGroup, "props"); + return _jisdict(pProps) ? pProps : pGroup; +} + +/* Count and fill share one walk. With pSrc NULL only the counts are + * wanted. */ +typedef struct { + size_t uGroup; size_t uProp; size_t uCtrl; + DasSrc* pSrc; const DasJdo* pParams; bool bFail; +} _walk_t; + +/* A property is settable when its set block names a protocol parameter. + * The counting pass and the fill pass must agree on this, so it is one + * function. */ +static const DasJdo* _setBlock(const DasJdo* pJdo) +{ + const DasJdo* pSet = _jget(pJdo, "set"); + return (_jisdict(pSet) && (_jstr(pSet, "param") != NULL)) ? pSet : NULL; +} + +static void _walkProp(_walk_t* pW, DasSrcGroup* pG, const char* sName, const DasJdo* pJdo) +{ + if(pW->pSrc == NULL){ + pW->uProp += 1; + if(_setBlock(pJdo) != NULL) pW->uCtrl += 1; + return; + } + if(pW->bFail) return; + + DasSrc* pS = pW->pSrc; + DasSrcProp* p = pS->aProp + pW->uProp; + memset(p, 0, sizeof(DasSrcProp)); + p->sec = pG->sec; + p->pGroup = pG; + p->sName = sName; + p->pJdo = pJdo; + + size_t uKey = strlen(g_sSecName[pG->sec]) + strlen(pG->sName) + strlen(sName) + 3; + p->sKey = (char*)malloc(uKey); + if(p->sKey == NULL){ pW->bFail = true; return; } + if(pG->sName[0] == '\0') + snprintf(p->sKey, uKey, "%s.%s", g_sSecName[pG->sec], sName); + else + snprintf(p->sKey, uKey, "%s.%s.%s", g_sSecName[pG->sec], pG->sName, sName); + + p->sValue = _scalarDup(_jget(pJdo, "value")); + + const DasJdo* pSet = _jget(pJdo, "set"); + if(_jisdict(pSet)){ + const char* sParam = _jstr(pSet, "param"); + if(sParam == NULL){ + daslog_warn_v("Property %s has a set block that names no parameter, " + "treating it as fixed", p->sKey); + } + else{ + /* Controls live in one block sized by the counting pass */ + if(pW->uCtrl >= pS->uCtrl){ pW->bFail = true; return; } + DasSrcCtrl* pC = pS->aCtrl + pW->uCtrl; + pW->uCtrl += 1; + memset(pC, 0, sizeof(DasSrcCtrl)); + pC->pProp = p; + pC->pSet = pSet; + pC->sParam = sParam; + pC->pParam = _jget(pW->pParams, sParam); + pC->bRequired = _jtrue(_jget(pSet, "required")) || _jtrue(_jget(pC->pParam, "required")); + p->pCtrl = pC; + if(!_fillEnum(pC)){ pW->bFail = true; return; } + _fillRange(pC); + } + } + _resolveType(p); + + if(_das_src_propEq(sName, "enabled") && (pG->nEnable < 0)){ + pG->nEnable = (int)(pW->uProp - pG->uFirst); + pG->sXor = _jstr(pJdo, "xorGroup"); + } + pW->uProp += 1; + pG->uCount += 1; +} + +static void _walkGroup(_walk_t* pW, das_src_sec_e sec, const char* sName, const DasJdo* pJdo) +{ + if(!_jisdict(pJdo)) return; + DasSrcGroup* pG = NULL; + if(pW->pSrc != NULL){ + pG = pW->pSrc->aGroup + pW->uGroup; + memset(pG, 0, sizeof(DasSrcGroup)); + pG->pSrc = pW->pSrc; + pG->sec = sec; + pG->sName = sName; + pG->pJdo = pJdo; + pG->uFirst = pW->uProp; + pG->nEnable = -1; + + const DasJdo* pMime = _jget(pJdo, "mimeTypes"); + if(_jisary(pMime) && (_jlen(pMime) > 0)){ + pG->aMime = (const char**)calloc(_jlen(pMime), sizeof(char*)); + if(pG->aMime == NULL){ pW->bFail = true; return; } + for(const das_json_ary_el* pEl = DasJdo_aryFirst(pMime); pEl; pEl = pEl->next){ + if(DasJdo_string(pEl->value) == NULL) continue; + pG->aMime[pG->uMime] = DasJdo_string(pEl->value); + pG->uMime += 1; + } + } + pW->pSrc->aSecCount[sec] += 1; + } + pW->uGroup += 1; + + const DasJdo* pProps = _groupProps(pJdo); + for(const das_json_dict_el* pEl = DasJdo_dictFirst(pProps); pEl; pEl = pEl->next){ + if(!_isPropDict(pEl->value)) continue; + _walkProp(pW, pG, pEl->name->string, pEl->value); + } +} + +/* Groups of a section in presentation order: "time" first for coordinates, + * otherwise as the node lists them. Forms want time on top and nodes do + * not always put it there. */ +static void _walkSection(_walk_t* pW, das_src_sec_e sec, const DasJdo* pSec) +{ + if(!_jisdict(pSec)) return; + if(pW->pSrc != NULL) pW->pSrc->aSecFirst[sec] = pW->uGroup; + if(sec == das_src_sec_coord){ + for(const das_json_dict_el* pEl = DasJdo_dictFirst(pSec); pEl; pEl = pEl->next) + if(_strieq(pEl->name->string, "time")) _walkGroup(pW, sec, pEl->name->string, pEl->value); + } + for(const das_json_dict_el* pEl = DasJdo_dictFirst(pSec); pEl; pEl = pEl->next){ + if((sec == das_src_sec_coord) && _strieq(pEl->name->string, "time")) continue; + _walkGroup(pW, sec, pEl->name->string, pEl->value); + } +} + +static void _walkIface(_walk_t* pW, const DasJdo* pIface) +{ + const DasJdo* pCoords = _jget(pIface, "coords"); + if(pCoords == NULL) pCoords = _jget(pIface, "coordinates"); + _walkSection(pW, das_src_sec_coord, pCoords); + _walkSection(pW, das_src_sec_data, _jget(pIface, "data")); + _walkSection(pW, das_src_sec_format, _jget(pIface, "formats")); + + /* Options are one group with no enclosing section */ + const DasJdo* pOpts = _jget(pIface, "options"); + if(_jisdict(pOpts)){ + if(pW->pSrc != NULL) pW->pSrc->aSecFirst[das_src_sec_option] = pW->uGroup; + _walkGroup(pW, das_src_sec_option, "", pOpts); + } +} + +/* ************************************************************************* */ +/* Examples, parsed into requisitions when the source is built */ + +/* "coords/time/props/min" -> "coords.time.min". The section token is + * folded to the current spelling so a saved requisition reads the same + * whatever node it came from. */ +static bool _exPathToKey(const char* sPath, char* sKey, size_t uLen) +{ + size_t u = 0; + int nTok = 0; + const char* pBeg = sPath; + sKey[0] = '\0'; + while(*pBeg){ + const char* pEnd = strchr(pBeg, '/'); + size_t n = pEnd ? (size_t)(pEnd - pBeg) : strlen(pBeg); + char sTok[DAS_MAX_ID_BUFSZ]; + if((n == 0)||(n >= sizeof(sTok))) return false; + memcpy(sTok, pBeg, n); sTok[n] = '\0'; + const char* sOut = sTok; + das_src_sec_e sec; + if(strcmp(sTok, "props") == 0) sOut = NULL; + else if((nTok == 0) && _das_src_secByName(sTok, &sec)) sOut = g_sSecName[sec]; + if(sOut != NULL){ + if(u + strlen(sOut) + 2 >= uLen) return false; + u = _catf(sKey, uLen, u, "%s%s", nTok ? "." : "", sOut); + ++nTok; + } + if(pEnd == NULL) break; + pBeg = pEnd + 1; + } + return nTok > 0; +} + +/* The flag value a control sends, without any prefix: the parameter's + * flags entry for it, or the flag name itself */ +static const char* _ctrlFlagVal(const DasSrcCtrl* pC, const char* sFlag, char* sBuf, size_t uLen) +{ + char sPath[DAS_MAX_ID_BUFSZ + 8]; + snprintf(sPath, sizeof(sPath), "flags/%s", sFlag); + const DasJdo* pF = _jget(pC->pParam, sPath); + if(!_das_json_scalar(_jget(pF, "value"), sBuf, uLen)) _strcpyz(sBuf, sFlag, uLen); + return sBuf; +} + +/* A 2018 example is raw protocol parameters. Each is mapped back onto + * the control that would have produced it, so the requisition looks the + * same as one from a current node. Parameters no control explains are + * kept as raw parameters, nothing is dropped. */ +static bool _exampleFromParams(DasSrc* pThis, DasReq* pReq, const DasJdo* pParams) +{ + char sVal[SCALAR_SZ], sFlagVal[SCALAR_SZ], sPVal[SCALAR_SZ]; + for(const das_json_dict_el* pP = DasJdo_dictFirst(pParams); pP; pP = pP->next){ + const char* sName = pP->name->string; + if(!_das_json_scalar(pP->value, sVal, sizeof(sVal))) continue; + + /* the controls feeding this parameter */ + bool bFlagSet = false, bHandled = false; + for(size_t u = 0; u < pThis->uProp; ++u){ + const DasSrcCtrl* pC = pThis->aProp[u].pCtrl; + if((pC == NULL)||(strcmp(pC->sParam, sName) != 0)) continue; + if(_jstr(pC->pSet, "flag") != NULL){ bFlagSet = true; break; } + } + + if(bFlagSet){ + const char* sSep = NULL; + for(size_t u = 0; (sSep == NULL) && (u < pThis->uProp); ++u){ + const DasSrcCtrl* pC = pThis->aProp[u].pCtrl; + if((pC == NULL)||(strcmp(pC->sParam, sName) != 0)) continue; + sSep = _jstr(pC->pParam, "flagSep"); + if(sSep == NULL) sSep = _jstr(pC->pParam, "flag_sep"); + } + if((sSep == NULL)||(sSep[0] == '\0')) sSep = " "; + + char sLeft[SCALAR_SZ] = {'\0'}; /* tokens no control owns */ + char* pTok = sVal; + while(pTok && *pTok){ + char* pNext = strstr(pTok, sSep); + if(pNext){ *pNext = '\0'; pNext += strlen(sSep); } + bool bFound = false; + for(size_t u = 0; !bFound && (u < pThis->uProp); ++u){ + const DasSrcCtrl* pC = pThis->aProp[u].pCtrl; + if((pC == NULL)||(strcmp(pC->sParam, sName) != 0)) continue; + const char* sFlag = _jstr(pC->pSet, "flag"); + if(sFlag == NULL) continue; + _ctrlFlagVal(pC, sFlag, sFlagVal, sizeof(sFlagVal)); + if(_strieq(sFlagVal, pTok)){ + if(DasReq_set(pReq, pThis->aProp[u].sKey, "true") != DAS_OKAY) return false; + bFound = true; + } + } + if(!bFound && (pTok[0] != '\0')){ + size_t n = strlen(sLeft); + snprintf(sLeft + n, sizeof(sLeft) - n, "%s%s", n ? sSep : "", pTok); + } + pTok = pNext; + } + if(sLeft[0] != '\0') + if(DasReq_setRaw(pReq, sName, sLeft) != DAS_OKAY) return false; + continue; + } + + for(size_t u = 0; !bHandled && (u < pThis->uProp); ++u){ + const DasSrcProp* p = pThis->aProp + u; + const DasSrcCtrl* pC = p->pCtrl; + if((pC == NULL)||(strcmp(pC->sParam, sName) != 0)) continue; + + const DasJdo* pEnum = _jget(pC->pSet, "enum"); + if(_jisary(pEnum)){ + /* which choice sends this protocol value? */ + for(const das_json_ary_el* pEl = DasJdo_aryFirst(pEnum); !bHandled && pEl; pEl = pEl->next){ + const DasJdo* pItem = _jisdict(pEl->value) ? _jget(pEl->value, "value") : pEl->value; + char sItem[SCALAR_SZ]; + if(!_das_json_scalar(pItem, sItem, sizeof(sItem))) continue; + bool bHit = _das_json_scalar(_jget(pEl->value, "pval"), sPVal, sizeof(sPVal)) ? + _strieq(sPVal, sVal) : _strieq(sItem, sVal); + if(bHit){ + if(DasReq_set(pReq, p->sKey, sItem) != DAS_OKAY) return false; + bHandled = true; + } + } + continue; + } + if(_das_json_scalar(_jget(pC->pSet, "pval"), sPVal, sizeof(sPVal))){ + /* a fixed protocol value stands for one state of a toggle */ + if(_strieq(sPVal, sVal)){ + bool bOn = true; + const DasJdo* pTrig = _jget(pC->pSet, "value"); + if(_jisbool(pTrig)) bOn = _jtrue(pTrig); + if(DasReq_set(pReq, p->sKey, bOn ? "true" : "false") != DAS_OKAY) return false; + bHandled = true; + } + continue; + } + if(DasReq_set(pReq, p->sKey, sVal) != DAS_OKAY) return false; + bHandled = true; + } + if(!bHandled) + if(DasReq_setRaw(pReq, sName, sVal) != DAS_OKAY) return false; + } + return true; +} + +static bool _loadExamples(DasSrc* pThis) +{ + const DasJdo* pEx = _jget(pThis->pIface, "examples"); + const DasJdo* pProto = _jget(pThis->pDoc, "protocol"); + bool bLegacyEx = false; + if(!_jisary(pEx)){ + pEx = _jget(pProto, "examples"); + bLegacyEx = _jisdict(pEx); + if(!bLegacyEx) return true; + } + size_t uMax = _jlen(pEx); + if(uMax == 0) return true; + pThis->aExample = (DasReq**)calloc(uMax, sizeof(DasReq*)); + if(pThis->aExample == NULL) return false; + + char sKey[256], sVal[SCALAR_SZ]; + if(bLegacyEx){ + for(const das_json_dict_el* pEl = DasJdo_dictFirst(pEx); pEl; pEl = pEl->next){ + DasReq* pReq = new_DasReq(); + if(pReq == NULL) return false; + pThis->aExample[pThis->uExample] = pReq; + pThis->uExample += 1; + const char* sLabel = _jstr(pEl->value, "title"); + if(sLabel == NULL) sLabel = _jstr(pEl->value, "name"); + if(sLabel == NULL) sLabel = pEl->name->string; + DasReq_setLabel(pReq, sLabel); + DasSrc_stamp(pThis, pReq); + const DasJdo* pParams = _jget(pEl->value, "http_params"); + if(_jisdict(pParams) && !_exampleFromParams(pThis, pReq, pParams)) return false; + } + return true; + } + + for(const das_json_ary_el* pEl = DasJdo_aryFirst(pEx); pEl; pEl = pEl->next){ + DasReq* pReq = new_DasReq(); + if(pReq == NULL) return false; + pThis->aExample[pThis->uExample] = pReq; + pThis->uExample += 1; + const char* sLabel = _jstr(pEl->value, "label"); + if(sLabel == NULL) sLabel = _jstr(pEl->value, "title"); + if(sLabel == NULL) sLabel = _jstr(pEl->value, "name"); + DasReq_setLabel(pReq, sLabel); + DasSrc_stamp(pThis, pReq); + const DasJdo* pSettings = _jget(pEl->value, "settings"); + for(const das_json_dict_el* pP = DasJdo_dictFirst(pSettings); pP; pP = pP->next){ + if(!_exPathToKey(pP->name->string, sKey, sizeof(sKey))){ + daslog_warn_v("Ignoring example setting '%s' in %s, the path makes no sense", + pP->name->string, pThis->sUrl); + continue; + } + if(!_das_json_scalar(pP->value, sVal, sizeof(sVal))) continue; + if(DasReq_set(pReq, sKey, sVal) != DAS_OKAY) return false; + } + } + return true; +} + +/* ************************************************************************* */ +/* Life cycle */ + +void del_DasSrc(DasSrc* pThis) +{ + if(pThis == NULL) return; + for(size_t u = 0; u < pThis->uExample; ++u) del_DasReq(pThis->aExample[u]); + free(pThis->aExample); + for(size_t u = 0; u < pThis->uProp; ++u){ + free(pThis->aProp[u].sKey); + free(pThis->aProp[u].sValue); + _clearCtrl(pThis->aProp[u].pCtrl); + } + free(pThis->aProp); + free(pThis->aCtrl); + for(size_t u = 0; u < pThis->uGroup; ++u) free(pThis->aGroup[u].aMime); + free(pThis->aGroup); + free(pThis->pDoc); + free(pThis->sPath); + free(pThis->sUrl); + free(pThis); +} + +DasSrc* new_DasSrc_json(const char* sDoc, size_t uLen, const char* sPathUri, const char* sUrl) +{ + const char* sWhere = sUrl ? sUrl : (sPathUri ? sPathUri : "(memory)"); + DasJdo* pDoc = das_json_parse(sDoc, uLen); + if(!_jisdict(pDoc)){ + daslog_error_v("Source document %s is not a JSON object", sWhere); + free(pDoc); + return NULL; + } + const char* sType = _jstr(pDoc, "type"); + if(!_strieq(sType, "HttpStreamSrc")){ + daslog_error_v("Node %s is of type %s, only HttpStreamSrc nodes describe data sources " + "this library can query", sWhere, sType ? sType : "(none)"); + free(pDoc); + return NULL; + } + + DasStreamSrc* pStream = (DasStreamSrc*)calloc(1, sizeof(DasStreamSrc)); + if(pStream == NULL){ free(pDoc); return NULL; } + DasSrc* pThis = &(pStream->base); + pThis->kind = das_src_kind_stream; + pThis->pDoc = pDoc; + + /* Every source has a URI, see D2URI_LOCAL. One from a bare document + * with no location is "local:/memory", which is all that can be said */ + char sUri[600]; + if((sPathUri != NULL) && (sPathUri[0] != '\0')) + _das_uri_expand(sPathUri, sUri, sizeof(sUri)); + else if((sUrl != NULL) && (sUrl[0] != '\0')) + _das_uri_local(sUrl, sUri, sizeof(sUri)); + else + strcpy(sUri, D2URI_LOCAL "/memory"); + pThis->sPath = _strdupz(sUri); + pThis->sUrl = _strdupz(sUrl); + if((pThis->sPath == NULL)||(pThis->sUrl == NULL)){ del_DasSrc(pThis); return NULL; } + + pThis->pIface = _jget(pDoc, "interface"); + if(!_jisdict(pThis->pIface)) pThis->pIface = NULL; + + pStream->pProto = _jget(pDoc, "protocol"); + if(!_jisdict(pStream->pProto)) pStream->pProto = NULL; + pStream->pParams = _jget(pStream->pProto, "httpParams"); + if(pStream->pParams == NULL) pStream->pParams = _jget(pStream->pProto, "http_params"); + pStream->pBaseUrls = _jget(pStream->pProto, "baseUrls"); + if(pStream->pBaseUrls == NULL) pStream->pBaseUrls = _jget(pStream->pProto, "base_urls"); + if(!_jisary(pStream->pBaseUrls)) pStream->pBaseUrls = NULL; + pThis->bLegacy = (_jget(pStream->pProto, "http_params") != NULL) || + (_jget(pThis->pIface, "coordinates") != NULL); + + /* Two passes over the same walk: count, allocate exactly that many, + * fill. Groups, properties and controls are three blocks, so a source + * is a handful of allocations however large its interface, which + * matters on a heap that never shrinks, as in a browser build. */ + _walk_t w = {0, 0, 0, NULL, pStream->pParams, false}; + _walkIface(&w, pThis->pIface); + pThis->aGroup = (DasSrcGroup*)calloc(w.uGroup ? w.uGroup : 1, sizeof(DasSrcGroup)); + pThis->aProp = (DasSrcProp*)calloc(w.uProp ? w.uProp : 1, sizeof(DasSrcProp)); + pThis->aCtrl = (DasSrcCtrl*)calloc(w.uCtrl ? w.uCtrl : 1, sizeof(DasSrcCtrl)); + if((pThis->aGroup == NULL)||(pThis->aProp == NULL)||(pThis->aCtrl == NULL)){ + del_DasSrc(pThis); + return NULL; + } + size_t uGroups = w.uGroup, uProps = w.uProp, uCtrls = w.uCtrl; + pThis->uCtrl = uCtrls; /* the fill pass checks against it */ + + w.uGroup = 0; w.uProp = 0; w.uCtrl = 0; w.pSrc = pThis; + _walkIface(&w, pThis->pIface); + pThis->uGroup = w.uGroup; + pThis->uProp = w.uProp; + pThis->uCtrl = w.uCtrl; + if(w.bFail || (uGroups != w.uGroup) || (uProps != w.uProp) || (uCtrls != w.uCtrl)){ + daslog_error_v("Couldn't build the interface of source %s", sWhere); + del_DasSrc(pThis); + return NULL; + } + + if(!_loadExamples(pThis)){ + daslog_error_v("Couldn't read the examples of source %s", sWhere); + del_DasSrc(pThis); + return NULL; + } + return pThis; +} + +DasSrc* new_DasSrc_node(const DasNode* pNode) +{ + if(pNode == NULL) return NULL; + if(!DasNode_isStreamSrc(pNode)){ + daslog_error_v("Node %s is not a stream source", DasNode_srcUrl(pNode)); + return NULL; + } + const DasJdo* pDom = DasNode_getJdo(pNode, NULL); + if(pDom == NULL) return NULL; + + /* A serialized copy keeps the source independent of the tree. Nodes + are a few KB, so this is cheaper than sharing with a reference count + would be to get right. */ + size_t uLen = 0; + char* sText = (char*)DasJdo_writeMinified(pDom, &uLen); + if(sText == NULL){ + daslog_error_v("Couldn't copy the document of node %s", DasNode_srcUrl(pNode)); + return NULL; + } + /* the reported size counts the terminator, which the parser rejects */ + DasSrc* pThis = new_DasSrc_json(sText, strlen(sText), DasNode_pathUri(pNode), DasNode_srcUrl(pNode)); + free(sText); + return pThis; +} + +/* ************************************************************************* */ +/* Identity */ + +das_src_kind_e DasSrc_kind(const DasSrc* pThis){ return pThis->kind; } +const char* DasSrc_path(const DasSrc* pThis){ return pThis->sPath; } +const char* DasSrc_url(const DasSrc* pThis){ return pThis->sUrl; } + +static const char* _docStr(const DasSrc* pThis, const char* sPath) +{ + const char* s = _jstr(pThis->pDoc, sPath); + return s ? s : ""; +} + +const char* DasSrc_name(const DasSrc* pThis) +{ + const char* s = _jstr(pThis->pDoc, "label"); + if(s == NULL) s = _jstr(pThis->pDoc, "name"); + return s ? s : ""; +} + +const char* DasSrc_title(const DasSrc* pThis){ return _docStr(pThis, "title"); } +const char* DasSrc_version(const DasSrc* pThis){ return _docStr(pThis, "version"); } + +bool DasSrc_needsAuth(const DasSrc* pThis) +{ + const DasJdo* pProto = _jget(pThis->pDoc, "protocol"); + const DasJdo* pAuth = _jget(pProto, "authorization"); + if(pAuth == NULL) pAuth = _jget(pProto, "authentication"); + return _jtrue(_jget(pAuth, "required")); +} + +static const DasJdo* _contacts(const DasSrc* pThis) +{ + const DasJdo* p = _jget(pThis->pDoc, "contacts"); + if(!_jisary(p)) p = _jget(pThis->pDoc, "tech_contacts"); + return _jisary(p) ? p : NULL; +} + +size_t DasSrc_contactCount(const DasSrc* pThis){ return _jlen(_contacts(pThis)); } + +const char* DasSrc_contact(const DasSrc* pThis, size_t uIdx, const char** psEmail) +{ + const das_json_ary_el* pEl = DasJdo_aryFirst(_contacts(pThis)); + for(size_t u = 0; (u < uIdx) && pEl; ++u) pEl = pEl->next; + if(pEl == NULL) return NULL; + const char* sName = _jstr(pEl->value, "name"); + const char* sEmail = _jstr(pEl->value, "email"); + if(psEmail) *psEmail = sEmail ? sEmail : ""; + return sName ? sName : ""; +} + +const DasJdo* DasSrc_jdo(const DasSrc* pThis, const char* sFragment) +{ + if((sFragment == NULL)||(sFragment[0] == '\0')) return pThis->pDoc; + return DasJdo_get(pThis->pDoc, sFragment); +} + +/* ************************************************************************* */ +/* Groups */ + +size_t DasSrc_groupCount(const DasSrc* pThis, das_src_sec_e sec) +{ + return ((sec >= 0)&&(sec < das_src_sec_size)) ? pThis->aSecCount[sec] : 0; +} + +const DasSrcGroup* DasSrc_group(const DasSrc* pThis, das_src_sec_e sec, size_t uIdx) +{ + if(uIdx >= DasSrc_groupCount(pThis, sec)) return NULL; + return pThis->aGroup + pThis->aSecFirst[sec] + uIdx; +} + +const DasSrcGroup* DasSrc_findGroup(const DasSrc* pThis, das_src_sec_e sec, const char* sName) +{ + if(sName == NULL) sName = ""; + for(size_t u = 0; u < DasSrc_groupCount(pThis, sec); ++u){ + const DasSrcGroup* pG = DasSrc_group(pThis, sec, u); + if(_strieq(pG->sName, sName)) return pG; + } + return NULL; +} + +das_src_sec_e DasSrcGroup_section(const DasSrcGroup* pThis){ return pThis->sec; } +const char* DasSrcGroup_name(const DasSrcGroup* pThis){ return pThis->sName; } + +const char* DasSrcGroup_label(const DasSrcGroup* pThis) +{ + const char* s = _jstr(pThis->pJdo, "label"); + if(s == NULL) s = _jstr(pThis->pJdo, "name"); + return s ? s : pThis->sName; +} + +const char* DasSrcGroup_title(const DasSrcGroup* pThis) +{ + const char* s = _jstr(pThis->pJdo, "title"); + return s ? s : ""; +} + +const DasSrcProp* DasSrcGroup_prop(const DasSrcGroup* pThis, size_t uIdx) +{ + return (uIdx < pThis->uCount) ? pThis->pSrc->aProp + pThis->uFirst + uIdx : NULL; +} + +const DasSrcProp* DasSrcGroup_enable(const DasSrcGroup* pThis) +{ + return (pThis->nEnable < 0) ? NULL : DasSrcGroup_prop(pThis, (size_t)pThis->nEnable); +} + +const char* DasSrcGroup_xorGroup(const DasSrcGroup* pThis){ return pThis->sXor; } +size_t DasSrcGroup_mimeCount(const DasSrcGroup* pThis){ return pThis->uMime; } + +const char* DasSrcGroup_mime(const DasSrcGroup* pThis, size_t uIdx) +{ + return (uIdx < pThis->uMime) ? pThis->aMime[uIdx] : NULL; +} + +size_t DasSrcGroup_propCount(const DasSrcGroup* pThis){ return pThis->uCount; } + +const DasSrcProp* DasSrcGroup_findProp(const DasSrcGroup* pThis, const char* sName) +{ + for(size_t u = 0; u < pThis->uCount; ++u){ + const DasSrcProp* p = DasSrcGroup_prop(pThis, u); + if(_das_src_propEq(p->sName, sName)) return p; + } + return NULL; +} + +size_t DasSrcGroup_ctrlCount(const DasSrcGroup* pThis) +{ + size_t n = 0; + for(size_t u = 0; u < pThis->uCount; ++u) + if(DasSrcGroup_prop(pThis, u)->pCtrl != NULL) ++n; + return n; +} + +/* ************************************************************************* */ +/* Properties */ + +size_t DasSrc_propCount(const DasSrc* pThis){ return pThis->uProp; } + +const DasSrcProp* DasSrc_prop(const DasSrc* pThis, size_t uIdx) +{ + return (uIdx < pThis->uProp) ? pThis->aProp + uIdx : NULL; +} + +das_src_sec_e DasSrcProp_section(const DasSrcProp* pThis){ return pThis->sec; } +const DasSrcGroup* DasSrcProp_group(const DasSrcProp* pThis){ return pThis->pGroup; } +const char* DasSrcProp_name(const DasSrcProp* pThis){ return pThis->sName; } +const char* DasSrcProp_key(const DasSrcProp* pThis){ return pThis->sKey; } +das_srcprop_type_e DasSrcProp_type(const DasSrcProp* pThis){ return pThis->type; } +const char* DasSrcProp_typeName(const DasSrcProp* pThis){ return pThis->sTypeName; } +const char* DasSrcProp_value(const DasSrcProp* pThis){ return pThis->sValue; } +const DasSrcCtrl* DasSrcProp_ctrl(const DasSrcProp* pThis){ return pThis->pCtrl; } + +/* Units for display. Current nodes put a units string on the property; + * both layouts may carry a units property with a "value" at the group + * level. */ +const char* DasSrcProp_units(const DasSrcProp* pThis) +{ + const DasJdo* pU = _jget(pThis->pJdo, "units"); + if(pU != NULL){ + if(pU->type == das_json_type_str) return DasJdo_string(pU); + if(_jstr(pU, "value")) return _jstr(pU, "value"); + } + if(pThis->sec == das_src_sec_coord){ + const char* s = _jstr(_groupProps(pThis->pGroup->pJdo), "units/value"); + if(s != NULL) return s; + } + if(pThis->type == das_srcprop_time) return "UTC"; + return ""; +} + +const char* DasSrcProp_title(const DasSrcProp* pThis, char* sBuf, size_t uLen) +{ + const char* s = _jstr(pThis->pJdo, "title"); + if(s == NULL) s = _jstr(pThis->pJdo, "summary"); + if(s == NULL) s = _jstr(pThis->pJdo, "label"); + if(s == NULL) s = _jstr(pThis->pJdo, "name"); + if(s != NULL) return s; + if(pThis->sec == das_src_sec_option) return ""; + + const char* sGroup = DasSrcGroup_label(pThis->pGroup); + const char* sProp = _das_src_propName(pThis->sName); + if(strcmp(sProp, "enabled") == 0){ + if(pThis->sec == das_src_sec_format){ + snprintf(sBuf, uLen, "Output as %s", sGroup); + } + else{ + /* A variable's title is usually the physical quantity, which + reads better after "Output" than its short label does */ + const char* sTitle = _jstr(pThis->pGroup->pJdo, "title"); + snprintf(sBuf, uLen, "Output %s", sTitle ? sTitle : sGroup); + } + return sBuf; + } + if(pThis->sec == das_src_sec_coord){ + const char* sFmt = NULL; + if(strcmp(sProp, "min") == 0) sFmt = "Minimum %s value to output"; + else if(strcmp(sProp, "max") == 0) sFmt = "Maximum %s value to output"; + else if(strcmp(sProp, "res") == 0) sFmt = "Average over %s bins of this width"; + else if(strcmp(sProp, "inter") == 0) sFmt = "Step between output %s values"; + else if(strcmp(sProp, "units") == 0) sFmt = "Units of %s values"; + if(sFmt != NULL){ + snprintf(sBuf, uLen, sFmt, sGroup); + return sBuf; + } + } + snprintf(sBuf, uLen, "%s %s", sGroup, pThis->sName); + return sBuf; +} + +/* ************************************************************************* */ +/* Controls */ + +const DasSrcProp* DasSrcCtrl_prop(const DasSrcCtrl* pThis){ return pThis->pProp; } +bool DasSrcCtrl_required(const DasSrcCtrl* pThis){ return pThis->bRequired; } +const char* DasSrcCtrl_param(const DasSrcCtrl* pThis){ return pThis->sParam; } +size_t DasSrcCtrl_enumCount(const DasSrcCtrl* pThis){ return pThis->uEnum; } + +const char* DasSrcCtrl_enum(const DasSrcCtrl* pThis, size_t uIdx) +{ + return (uIdx < pThis->uEnum) ? pThis->aEnum[uIdx] : NULL; +} + +bool DasSrcCtrl_range(const DasSrcCtrl* pThis, const char** psLo, const char** psHi) +{ + if((pThis->sLo == NULL)&&(pThis->sHi == NULL)) return false; + if(psLo) *psLo = pThis->sLo ? pThis->sLo : ""; + if(psHi) *psHi = pThis->sHi ? pThis->sHi : ""; + return true; +} + +DasErrCode DasSrcCtrl_check(const DasSrcCtrl* pThis, const char* sVal, char* sWhy, size_t uWhy) +{ + if(sVal == NULL) sVal = ""; + if(sWhy && uWhy) sWhy[0] = '\0'; + char sDummy[1]; + if(sWhy == NULL){ sWhy = sDummy; uWhy = 1; } + const char* sKey = pThis->pProp->sKey; + char* pEnd = NULL; + + switch(pThis->pProp->type){ + case das_srcprop_bool:{ + bool b; + if(_parseBool(sVal, &b)) return DAS_OKAY; + snprintf(sWhy, uWhy, "'%s' is not a boolean value for %s, use true or false", sVal, sKey); + return DASERR_SRC; + } + case das_srcprop_enum:{ + for(size_t u = 0; u < pThis->uEnum; ++u) + if(_strieq(pThis->aEnum[u], sVal)) return DAS_OKAY; + size_t u = snprintf(sWhy, uWhy, "'%s' is not an allowed value for %s, choose one of:", sVal, sKey); + for(size_t v = 0; v < pThis->uEnum; ++v) + u = _catf(sWhy, uWhy, u, "%s %s", v ? "," : "", pThis->aEnum[v]); + return DASERR_SRC; + } + case das_srcprop_int: + strtol(sVal, &pEnd, 10); + if((pEnd != sVal) && (*pEnd == '\0')) return DAS_OKAY; + snprintf(sWhy, uWhy, "'%s' is not an integer value for %s", sVal, sKey); + return DASERR_SRC; + case das_srcprop_real: + strtod(sVal, &pEnd); + if((pEnd != sVal) && (*pEnd == '\0')) return DAS_OKAY; + snprintf(sWhy, uWhy, "'%s' is not a real value for %s", sVal, sKey); + return DASERR_SRC; + case das_srcprop_time:{ + /* "now" is how nodes spell an open ended range and servers take + it in requests too */ + das_time dt; + if(_strieq(sVal, "now") || dt_parsetime(&dt, sVal)) return DAS_OKAY; + snprintf(sWhy, uWhy, "'%s' is not a time value for %s, use ISO 8601 such as " + "2026-01-15T12:00", sVal, sKey); + return DASERR_SRC; + } + default: + return DAS_OKAY; + } +} + +/* ************************************************************************* */ +/* Key lookup */ + +int _DasSrc_match(const DasSrc* pThis, const char* sKey, const DasSrcProp** aOut, int nMax) +{ + char aParts[3][DAS_MAX_ID_BUFSZ]; + int nParts = _das_src_splitKey(sKey, aParts); + if(nParts == 0) return 0; + + int n = 0; + das_src_sec_e sec; + const DasSrcProp* p = NULL; + +#define _ADD_MATCH if(n < nMax) aOut[n] = p; ++n; + + if(nParts == 3){ + if(!_das_src_secByName(aParts[0], &sec)) return 0; + for(size_t u = 0; u < pThis->uProp; ++u){ + p = pThis->aProp + u; + if((p->sec == sec) && _strieq(p->pGroup->sName, aParts[1]) && + _das_src_propEq(p->sName, aParts[2])){ _ADD_MATCH } + } + return n; + } + + if(nParts == 2){ + /* "option.X" is a section qualified option, "SEC.GROUP" toggles a + variable, anything else is GROUP.PROP in any section */ + if(_das_src_secByName(aParts[0], &sec)){ + for(size_t u = 0; u < pThis->uProp; ++u){ + p = pThis->aProp + u; + if(p->sec != sec) continue; + if((sec == das_src_sec_option) && _das_src_propEq(p->sName, aParts[1])){ _ADD_MATCH } + else if((sec != das_src_sec_option) && _strieq(p->pGroup->sName, aParts[1]) && + _das_src_propEq(p->sName, "enabled")){ _ADD_MATCH } + } + if(n > 0) return n; + } + for(size_t u = 0; u < pThis->uProp; ++u){ + p = pThis->aProp + u; + if(_strieq(p->pGroup->sName, aParts[0]) && _das_src_propEq(p->sName, aParts[1])){ _ADD_MATCH } + } + return n; + } + + /* One part: an option, or a variable/format toggle */ + for(size_t u = 0; u < pThis->uProp; ++u){ + p = pThis->aProp + u; + if((p->sec == das_src_sec_option) && _das_src_propEq(p->sName, aParts[0])){ _ADD_MATCH } + else if((p->sec != das_src_sec_option) && _strieq(p->pGroup->sName, aParts[0]) && + _das_src_propEq(p->sName, "enabled")){ _ADD_MATCH } + } + if(n > 0) return n; + + /* Last resort, a bare property name. Unique in practice for things + like 'serial', ambiguous for 'fracSecs' shared by das and csv, and + the ambiguity message then lists the full keys. */ + for(size_t u = 0; u < pThis->uProp; ++u){ + p = pThis->aProp + u; + if((p->sec != das_src_sec_option) && _das_src_propEq(p->sName, aParts[0])){ _ADD_MATCH } + } +#undef _ADD_MATCH + return n; +} + +const DasSrcGroup* _DasSrc_coordGroup(const DasSrc* pThis, const char* sKey) +{ + char aParts[3][DAS_MAX_ID_BUFSZ]; + int nParts = _das_src_splitKey(sKey, aParts); + const char* sGroup = NULL; + das_src_sec_e sec; + if(nParts == 1) sGroup = aParts[0]; + else if((nParts == 2) && _das_src_secByName(aParts[0], &sec) && (sec == das_src_sec_coord)) + sGroup = aParts[1]; + else return NULL; + return DasSrc_findGroup(pThis, das_src_sec_coord, sGroup); +} + +#define MAX_MATCH 8 + +static void _ambiguous(const char* sKey, const DasSrcProp** aMatch, int n, char* sWhy, size_t uWhy) +{ + size_t u = snprintf(sWhy, uWhy, "Key '%s' is ambiguous, use one of:", sKey); + for(int i = 0; (i < n) && (i < MAX_MATCH); ++i) + u = _catf(sWhy, uWhy, u, " %s", aMatch[i]->sKey); +} + +const DasSrcProp* DasSrc_findProp(const DasSrc* pThis, const char* sKey, char* sWhy, size_t uWhy) +{ + char sDummy[1]; + if(sWhy == NULL){ sWhy = sDummy; uWhy = 1; } + sWhy[0] = '\0'; + const DasSrcProp* aMatch[MAX_MATCH]; + int n = _DasSrc_match(pThis, sKey, aMatch, MAX_MATCH); + if(n == 1) return aMatch[0]; + if(n == 0) snprintf(sWhy, uWhy, "No property named '%s' in this source", sKey); + else _ambiguous(sKey, aMatch, n, sWhy, uWhy); + return NULL; +} + +/* ************************************************************************* */ +/* Requisitions */ + +size_t DasSrc_exampleCount(const DasSrc* pThis){ return pThis->uExample; } + +const char* DasSrc_exampleLabel(const DasSrc* pThis, size_t uIdx) +{ + return (uIdx < pThis->uExample) ? DasReq_label(pThis->aExample[uIdx]) : NULL; +} + +void DasSrc_stamp(const DasSrc* pThis, DasReq* pReq) +{ + DasReq_setSrc(pReq, pThis->sPath, pThis->sUrl, DasSrc_version(pThis)); +} + +DasReq* DasSrc_newReq(const DasSrc* pThis, int nExample) +{ + if(nExample >= 0){ + if((size_t)nExample >= pThis->uExample){ + daslog_error_v("Source %s has %zu example(s), there is no example %d", + pThis->sUrl, pThis->uExample, nExample); + return NULL; + } + return copy_DasReq(pThis->aExample[nExample]); + } + DasReq* pReq = new_DasReq(); + if(pReq == NULL) return NULL; + DasSrc_stamp(pThis, pReq); + for(size_t u = 0; u < pThis->uProp; ++u){ + const DasSrcProp* p = pThis->aProp + u; + if((p->pCtrl == NULL)||!p->pCtrl->bRequired||(p->sValue != NULL)) continue; + if(DasReq_set(pReq, p->sKey, "") != DAS_OKAY){ del_DasReq(pReq); return NULL; } + } + return pReq; +} + +/* Identity is the catalog URI alone. The URL is where one copy of the + * node happened to come from; a mirror or a local mount of the same node + * has a different one and must validate the same requisitions. */ +static bool _stampMatches(const DasSrc* pThis, const DasReq* pReq) +{ + const char* sPath = DasReq_srcPath(pReq); + if(sPath[0] == '\0') return true; /* unstamped */ + return _strieq(sPath, pThis->sPath); +} + +/* One item of a coordinate range: find the control, check the value, + * add the derived entry. Returns false with sWhy filled on any trouble. */ +static bool _rangeItem( + const DasSrc* pThis, DasReq* pReq, const DasSrcGroup* pG, const char* sProp, + const char* sVal, bool* aPresent, bool* aCovered, char* sWhy, size_t uWhy +){ + const DasSrcProp* p = DasSrcGroup_findProp(pG, sProp); + if((p == NULL)||(p->pCtrl == NULL)){ + snprintf(sWhy, uWhy, "Coordinate %s has no settable '%s' property", pG->sName, sProp); + return false; + } + if(DasSrcCtrl_check(p->pCtrl, sVal, sWhy, uWhy) != DAS_OKAY) return false; + int n = _DasReq_add(pReq, p->sKey, sVal, true); + if(n < 0){ snprintf(sWhy, uWhy, "Out of memory"); return false; } + size_t uProp = (size_t)(p - pThis->aProp); + pReq->aEntry[n].nProp = (int)uProp; + _DasReq_setVerdict(pReq, (size_t)n, das_req_ok, NULL); + aPresent[uProp] = true; + aCovered[uProp] = true; + return true; +} + +/* GROUP=MIN,MAX[,STEP] or GROUP=SPAN, see "Keys" in source.h */ +static bool _rangeEntry( + const DasSrc* pThis, DasReq* pReq, const DasSrcGroup* pG, const char* sVal, + bool* aPresent, bool* aCovered, char* sWhy, size_t uWhy +){ + if(strchr(sVal, ',') == NULL){ + /* a single time denotes its span */ + const DasSrcProp* pMin = DasSrcGroup_findProp(pG, "min"); + if((pMin == NULL)||(pMin->type != das_srcprop_time)){ + snprintf(sWhy, uWhy, "Give coordinate %s a range as MIN,MAX", pG->sName); + return false; + } + das_time dtBeg, dtEnd; + if(!dt_parsespan(sVal, &dtBeg, &dtEnd)){ + snprintf(sWhy, uWhy, "'%s' is neither a MIN,MAX range nor a time span such " + "as 2026-01-02 or 2026-01-02T14", sVal); + return false; + } + char sBeg[40], sEnd[40]; + dt_isoc(&dtBeg, sBeg, sizeof(sBeg), 0); + dt_isoc(&dtEnd, sEnd, sizeof(sEnd), 0); + return _rangeItem(pThis, pReq, pG, "min", sBeg, aPresent, aCovered, sWhy, uWhy) && + _rangeItem(pThis, pReq, pG, "max", sEnd, aPresent, aCovered, sWhy, uWhy); + } + + const char* aProps[3] = {"min", "max", NULL}; + aProps[2] = DasSrcGroup_findProp(pG, "res") ? "res" : "inter"; + + char sCopy[SCALAR_SZ]; + _strcpyz(sCopy, sVal, sizeof(sCopy)); + char* pTok = sCopy; + for(int i = 0; i < 4; ++i){ + char* pComma = strchr(pTok, ','); + if(pComma) *pComma = '\0'; + if(i == 3){ + snprintf(sWhy, uWhy, "Too many values for coordinate %s, at most MIN,MAX,STEP", pG->sName); + return false; + } + if(pTok[0] != '\0') + if(!_rangeItem(pThis, pReq, pG, aProps[i], pTok, aPresent, aCovered, sWhy, uWhy)) + return false; + if(pComma == NULL) break; + pTok = pComma + 1; + } + return true; +} + +DasErrCode DasSrc_check(const DasSrc* pThis, DasReq* pReq, size_t* puBad) +{ + if(puBad) *puBad = 0; + if(!_stampMatches(pThis, pReq)){ + daslog_error_v("Requisition is for source %s%s, not %s%s", + DasReq_srcPath(pReq), DasReq_srcPath(pReq)[0] ? "" : DasReq_srcUrl(pReq), + pThis->sPath, pThis->sPath[0] ? "" : pThis->sUrl); + return DASERR_SRC; + } + _DasReq_dropDerived(pReq); + _DasReq_clearVerdicts(pReq); + + /* present: some entry names the property, even badly. covered: a good + non-empty value was given for it. */ + size_t uProp = pThis->uProp ? pThis->uProp : 1; + bool* aPresent = (bool*)calloc(uProp * 2, sizeof(bool)); + if(aPresent == NULL) return DASERR_SRC; + bool* aCovered = aPresent + uProp; + + size_t uBad = 0; + char sWhy[1024]; + size_t uClient = pReq->uEntry; /* derived entries are appended past here */ + for(size_t u = 0; u < uClient; ++u){ + const char* sKey = pReq->aEntry[u].sKey; + const char* sVal = pReq->aEntry[u].sVal; + + const DasSrcGroup* pG = _DasSrc_coordGroup(pThis, sKey); + if(pG != NULL){ + if(_rangeEntry(pThis, pReq, pG, sVal, aPresent, aCovered, sWhy, sizeof(sWhy))) + _DasReq_setVerdict(pReq, u, das_req_ok, NULL); + else{ + _DasReq_setVerdict(pReq, u, das_req_bad, sWhy); + ++uBad; + } + continue; + } + + const DasSrcProp* aMatch[MAX_MATCH]; + int n = _DasSrc_match(pThis, sKey, aMatch, MAX_MATCH); + if(n == 0){ + snprintf(sWhy, sizeof(sWhy), "No property named '%s' in this source", sKey); + _DasReq_setVerdict(pReq, u, das_req_bad, sWhy); ++uBad; continue; + } + if(n > 1){ + _ambiguous(sKey, aMatch, n, sWhy, sizeof(sWhy)); + _DasReq_setVerdict(pReq, u, das_req_bad, sWhy); ++uBad; continue; + } + const DasSrcProp* p = aMatch[0]; + size_t uP = (size_t)(p - pThis->aProp); + pReq->aEntry[u].nProp = (int)uP; + if(p->pCtrl == NULL){ + snprintf(sWhy, sizeof(sWhy), "%s is a fixed property of this source and cannot be set", p->sKey); + _DasReq_setVerdict(pReq, u, das_req_bad, sWhy); ++uBad; continue; + } + aPresent[uP] = true; + if(sVal[0] == '\0'){ + if(p->pCtrl->bRequired && (p->sValue == NULL)){ + snprintf(sWhy, sizeof(sWhy), "%s is required, give it a value", p->sKey); + _DasReq_setVerdict(pReq, u, das_req_bad, sWhy); ++uBad; + } + else + _DasReq_setVerdict(pReq, u, das_req_ok, NULL); + continue; + } + if(DasSrcCtrl_check(p->pCtrl, sVal, sWhy, sizeof(sWhy)) != DAS_OKAY){ + _DasReq_setVerdict(pReq, u, das_req_bad, sWhy); ++uBad; continue; + } + aCovered[uP] = true; + _DasReq_setVerdict(pReq, u, das_req_ok, NULL); + } + + /* An empty entry for a required control is fine after all when a range + entry covered the same control: the blank requisition lists + coords.time.min, the user typed time=A,B, both name one property */ + for(size_t u = 0; u < uClient; ++u){ + struct das_req_entry* pE = pReq->aEntry + u; + if((pE->verdict != das_req_bad)||(pE->nProp < 0)||(pE->sVal[0] != '\0')) continue; + if(!aCovered[pE->nProp]) continue; + _DasReq_setVerdict(pReq, u, das_req_ok, NULL); + --uBad; + } + + /* required controls nobody mentioned */ + for(size_t u = 0; u < pThis->uProp; ++u){ + const DasSrcProp* p = pThis->aProp + u; + if((p->pCtrl == NULL)||!p->pCtrl->bRequired||(p->sValue != NULL)||aPresent[u]) continue; + int n = _DasReq_add(pReq, p->sKey, "", true); + if(n < 0) break; + pReq->aEntry[n].nProp = (int)u; + snprintf(sWhy, sizeof(sWhy), "%s is required, set it with %s=VALUE", p->sKey, p->sKey); + _DasReq_setVerdict(pReq, (size_t)n, das_req_bad, sWhy); + ++uBad; + } + free(aPresent); + if(puBad) *puBad = uBad; + return uBad ? DASERR_SRC : DAS_OKAY; +} + +/* The format group a requisition turns on, else the node's default */ +static const DasSrcGroup* _chosenFormat(const DasSrc* pThis, const DasReq* pReq) +{ + const DasSrcGroup* pDef = NULL; + for(size_t g = 0; g < DasSrc_groupCount(pThis, das_src_sec_format); ++g){ + const DasSrcGroup* pG = DasSrc_group(pThis, das_src_sec_format, g); + const DasSrcProp* pOn = DasSrcGroup_enable(pG); + if(pOn == NULL) continue; + bool b = false; + if((pDef == NULL) && _parseBool(pOn->sValue, &b) && b) pDef = pG; + if(pReq == NULL) continue; + for(size_t u = 0; u < pReq->uEntry; ++u){ + if(pReq->aEntry[u].bDerived) continue; + const DasSrcProp* aMatch[1]; + if(_DasSrc_match(pThis, pReq->aEntry[u].sKey, aMatch, 1) != 1) continue; + if((aMatch[0] == pOn) && _parseBool(pReq->aEntry[u].sVal, &b) && b) return pG; + } + } + return pDef; +} + +const char* DasSrc_mime(const DasSrc* pThis, const DasReq* pReq) +{ + const DasSrcGroup* pG = _chosenFormat(pThis, pReq); + if((pG != NULL) && (pG->uMime > 0)) return pG->aMime[0]; + + /* 2018 nodes state a default format outside the interface */ + const char* s = _jstr(pThis->pDoc, "format/default/mime"); + if(s == NULL) s = _jstr(pThis->pDoc, "protocol/format/default/mime"); + if(s == NULL) s = _jstr(pThis->pDoc, "interface/format/default/mime"); + return s ? s : ""; +} + +/* ************************************************************************* */ +/* Stream sources: translating a requisition to protocol parameters */ + +static const DasStreamSrc* _stream(const DasSrc* pThis) +{ + return (pThis->kind == das_src_kind_stream) ? (const DasStreamSrc*)pThis : NULL; +} + +size_t DasStreamSrc_urlCount(const DasSrc* pThis) +{ + const DasStreamSrc* pS = _stream(pThis); + return pS ? _jlen(pS->pBaseUrls) : 0; +} + +const char* DasStreamSrc_baseUrl(const DasSrc* pThis, size_t uIdx) +{ + const DasStreamSrc* pS = _stream(pThis); + if(pS == NULL) return NULL; + const das_json_ary_el* pEl = DasJdo_aryFirst(pS->pBaseUrls); + for(size_t u = 0; (u < uIdx) && pEl; ++u) pEl = pEl->next; + return pEl ? DasJdo_string(pEl->value) : NULL; +} + +/* The output parameter list. Names are borrowed from the document or the + * requisition, values are owned because flag sets grow by appending. */ +typedef struct { const char* sName; char* sVal; } _param_t; +typedef struct { _param_t* a; size_t u; size_t uCap; } _out_t; + +static _param_t* _outFind(_out_t* pO, const char* sName) +{ + for(size_t u = 0; u < pO->u; ++u) + if(strcmp(pO->a[u].sName, sName) == 0) return pO->a + u; + return NULL; +} + +static bool _outSet(_out_t* pO, const char* sName, const char* sVal) +{ + _param_t* p = _outFind(pO, sName); + if(p == NULL){ + if(pO->u == pO->uCap){ + size_t uCap = pO->uCap ? pO->uCap * 2 : 16; + _param_t* a = (_param_t*)realloc(pO->a, uCap * sizeof(_param_t)); + if(a == NULL) return false; + pO->a = a; + pO->uCap = uCap; + } + p = pO->a + pO->u; + p->sName = sName; + p->sVal = NULL; + pO->u += 1; + } + char* sNew = _strdupz(sVal); + if(sNew == NULL) return false; + free(p->sVal); + p->sVal = sNew; + return true; +} + +static bool _outAddFlag(_out_t* pO, const char* sName, const char* sFlagVal, const char* sSep) +{ + _param_t* p = _outFind(pO, sName); + if((p == NULL)||(p->sVal[0] == '\0')) return _outSet(pO, sName, sFlagVal); + char* sNew = das_string("%s%s%s", p->sVal, sSep, sFlagVal); + if(sNew == NULL) return false; + free(p->sVal); + p->sVal = sNew; + return true; +} + +static void _outClear(_out_t* pO) +{ + for(size_t u = 0; u < pO->u; ++u) free(pO->a[u].sVal); + free(pO->a); + memset(pO, 0, sizeof(_out_t)); +} + +/* One control's value to protocol parameters, by its set block: + * + * - enum: the chosen item's "pval" if it has one, else its value, and + * an item may name its own flag + * - boolean: the block fires for one state only, "value" (true unless + * said otherwise); the other state is the default and sends nothing. + * With neither pval nor flag the state itself is sent as true/false. + * - flag: the flag's value (its entry under the parameter's flags, else + * the flag name), with the flag's prefix, appended to the parameter + * with the parameter's flagSep + * - otherwise the value as given, or the block's pval + */ +static bool _translate(const DasSrcCtrl* pC, const char* sVal, _out_t* pO, char* sWhy, size_t uWhy) +{ + const DasSrcProp* p = pC->pProp; + char sFlag[DAS_MAX_ID_BUFSZ] = {'\0'}; + char sPVal[SCALAR_SZ] = {'\0'}; + _strcpyz(sFlag, _jstr(pC->pSet, "flag"), sizeof(sFlag)); + bool bHavePVal = _das_json_scalar(_jget(pC->pSet, "pval"), sPVal, sizeof(sPVal)); + + const DasJdo* pEnum = _jget(pC->pSet, "enum"); + if(_jisary(pEnum)){ + char sItem[SCALAR_SZ]; + for(const das_json_ary_el* pEl = DasJdo_aryFirst(pEnum); pEl; pEl = pEl->next){ + const DasJdo* pItem = _jisdict(pEl->value) ? _jget(pEl->value, "value") : pEl->value; + if(!_das_json_scalar(pItem, sItem, sizeof(sItem))) continue; + if(!_strieq(sItem, sVal)) continue; + if(_das_json_scalar(_jget(pEl->value, "pval"), sPVal, sizeof(sPVal))) bHavePVal = true; + else if(!bHavePVal){ _strcpyz(sPVal, sItem, sizeof(sPVal)); bHavePVal = true; } + if(_jstr(pEl->value, "flag")) _strcpyz(sFlag, _jstr(pEl->value, "flag"), sizeof(sFlag)); + break; + } + } + else if(p->type == das_srcprop_bool){ + bool bUser = false; + _parseBool(sVal, &bUser); + const DasJdo* pTrig = _jget(pC->pSet, "value"); + if(_jisbool(pTrig) && (bUser != _jtrue(pTrig))) return true; + if(!bHavePVal && (sFlag[0] == '\0')){ + _strcpyz(sPVal, bUser ? "true" : "false", sizeof(sPVal)); + bHavePVal = true; + } + } + + if(sFlag[0] != '\0'){ + char sPath[DAS_MAX_ID_BUFSZ + 8]; + snprintf(sPath, sizeof(sPath), "flags/%s", sFlag); + const DasJdo* pF = _jget(pC->pParam, sPath); + if(pF == NULL){ + snprintf(sWhy, uWhy, "Flag '%s' for %s is not defined under protocol parameter %s, " + "the node is broken", sFlag, p->sKey, pC->sParam); + return false; + } + char sFVal[SCALAR_SZ], sFull[SCALAR_SZ]; + if(!_das_json_scalar(_jget(pF, "value"), sFVal, sizeof(sFVal))) _strcpyz(sFVal, sFlag, sizeof(sFVal)); + const char* sPrefix = _jstr(pF, "prefix"); + snprintf(sFull, sizeof(sFull), "%s%s", sPrefix ? sPrefix : "", sFVal); + const char* sSep = _jstr(pC->pParam, "flagSep"); + if(sSep == NULL) sSep = _jstr(pC->pParam, "flag_sep"); + if(sSep == NULL) sSep = " "; + if(!_outAddFlag(pO, pC->sParam, sFull, sSep)){ snprintf(sWhy, uWhy, "Out of memory"); return false; } + return true; + } + if(!_outSet(pO, pC->sParam, bHavePVal ? sPVal : sVal)){ snprintf(sWhy, uWhy, "Out of memory"); return false; } + return true; +} + +/* The value a requisition gives a property: the last good non-empty + * entry that resolved to it, derived or not */ +static const char* _reqValueFor(const DasReq* pReq, size_t uProp) +{ + const char* sVal = NULL; + for(size_t u = 0; u < pReq->uEntry; ++u){ + const struct das_req_entry* pE = pReq->aEntry + u; + if((pE->nProp != (int)uProp)||(pE->verdict != das_req_ok)||(pE->sVal[0] == '\0')) continue; + sVal = pE->sVal; + } + return sVal; +} + +static bool _sameAsDefault(const DasSrcProp* p, const char* sVal) +{ + if(p->sValue == NULL) return false; + if(p->type == das_srcprop_bool){ + bool bA = false, bB = false; + return _parseBool(sVal, &bA) && _parseBool(p->sValue, &bB) && (bA == bB); + } + return _strieq(sVal, p->sValue); +} + +size_t DasStreamSrc_url(const DasSrc* pThis, DasReq* pReq, size_t uBase, char* sBuf, size_t uLen) +{ + if(uLen > 0) sBuf[0] = '\0'; + const DasStreamSrc* pS = _stream(pThis); + if((pS == NULL)||(pS->pProto == NULL)||(pS->pBaseUrls == NULL)){ + daslog_error_v("Source %s has no protocol section with base URLs, it cannot be queried", + pThis->sUrl); + return 0; + } + const char* sBase = DasStreamSrc_baseUrl(pThis, uBase); + if(sBase == NULL){ + daslog_error_v("Source %s has %zu base URL(s), there is no number %zu", + pThis->sUrl, DasStreamSrc_urlCount(pThis), uBase); + return 0; + } + + size_t uBad = 0; + if(DasSrc_check(pThis, pReq, &uBad) != DAS_OKAY){ + for(size_t u = 0; u < pReq->uEntry; ++u){ + if(pReq->aEntry[u].verdict == das_req_bad){ + daslog_error_v("%s", pReq->aEntry[u].sWhy); + break; + } + } + return 0; + } + + _out_t out = {NULL, 0, 0}; + char sWhy[1024]; + for(size_t u = 0; u < pThis->uProp; ++u){ + const DasSrcProp* p = pThis->aProp + u; + const DasSrcCtrl* pC = p->pCtrl; + if(pC == NULL) continue; + + const char* sVal = _reqValueFor(pReq, u); + bool bDefault = false; + if(sVal == NULL){ + if(!pC->bRequired || (p->sValue == NULL)) continue; + sVal = p->sValue; + bDefault = true; + } + /* A required default fills a hole, it never replaces what another + control already put in a shared parameter */ + if(bDefault && _outFind(&out, pC->sParam)) continue; + + /* A value equal to the default carries no information unless the + server insists on hearing it */ + if(!bDefault && !pC->bRequired && _sameAsDefault(p, sVal)) continue; + + if(!_translate(pC, sVal, &out, sWhy, sizeof(sWhy))){ + daslog_error_v("%s", sWhy); + _outClear(&out); + return 0; + } + } + + for(size_t u = 0; u < pReq->uRaw; ++u){ + if(!_outSet(&out, pReq->aRaw[u].sName, pReq->aRaw[u].sVal)){ _outClear(&out); return 0; } + } + + /* Protocol parameters the node marks required, whatever supplies them */ + for(const das_json_dict_el* pEl = DasJdo_dictFirst(pS->pParams); pEl; pEl = pEl->next){ + if(!_jtrue(_jget(pEl->value, "required"))) continue; + const char* sName = pEl->name->string; + if(_outFind(&out, sName) != NULL) continue; + + size_t n = snprintf(sWhy, sizeof(sWhy), "Required parameter %s missing, set it with", sName); + int nKeys = 0; + for(size_t i = 0; i < pThis->uProp; ++i){ + const DasSrcCtrl* pC = pThis->aProp[i].pCtrl; + if(pC && (strcmp(pC->sParam, sName) == 0)){ + n = _catf(sWhy, sizeof(sWhy), n, "%s %s=VALUE", nKeys ? " or" : "", pThis->aProp[i].sKey); + ++nKeys; + } + } + if(nKeys == 0) _catf(sWhy, sizeof(sWhy), n, " a raw parameter, no interface key sets it"); + daslog_error_v("%s", sWhy); + _outClear(&out); + return 0; + } + + size_t u = _catf(sBuf, uLen, 0, "%s", sBase); + size_t uNeed = strlen(sBase); + const char* sJoin = strchr(sBase, '?') ? "&" : "?"; + for(size_t i = 0; i < out.u; ++i){ + size_t uEnc = das_url_encode(out.a[i].sVal, NULL, 0) + 1; + char* sEnc = (char*)malloc(uEnc); + if(sEnc == NULL){ _outClear(&out); return 0; } + das_url_encode(out.a[i].sVal, sEnc, uEnc); + u = _catf(sBuf, uLen, u, "%s%s=%s", sJoin, out.a[i].sName, sEnc); + uNeed += strlen(sJoin) + strlen(out.a[i].sName) + 1 + strlen(sEnc); + free(sEnc); + sJoin = "&"; + } + _outClear(&out); + return uNeed; +} + +/* ************************************************************************* */ +/* The default source under a node, used by DasCat_newSrc() */ + +DasNode* _DasSrc_defNode(DasNode* pNode) +{ + if(DasNode_isStreamSrc(pNode)) return pNode; + if(!DasNode_isCatalog(pNode)){ + daslog_error_v("Node %s is not a data source", DasNode_srcUrl(pNode)); + return NULL; + } + + size_t uKids = DasNode_subCount(pNode); + const char* sPick = NULL; + bool bHasCatalogs = false; + for(size_t u = 0; u < uKids; ++u){ + const char* sName = DasNode_subName(pNode, u); + das_node_type_e nType = DasNode_subType(pNode, u); + if((nType == d2node_catalog)||(nType == d2node_spdf_cat)||(nType == d2node_spase_cat)) + bHasCatalogs = true; + if(nType != d2node_stream_src) continue; + if(_strieq(sName, "flex")){ sPick = sName; break; } + if(sPick == NULL) sPick = sName; + } + if(sPick == NULL){ + if(bHasCatalogs) + daslog_error_v("%s is a catalog, not a data source, pick one of its children", + DasNode_srcUrl(pNode)); + else + daslog_error_v("No stream source children under %s", DasNode_srcUrl(pNode)); + return NULL; + } + + /* Credentials, agent and cache are inherited from the node's root */ + DasNode* pSub = DasNode_subNode(pNode, sPick, NULL, NULL); + if(pSub == NULL){ + daslog_error_v("Couldn't load source '%s' from %s", sPick, DasNode_srcUrl(pNode)); + return NULL; + } + if(!DasNode_isStreamSrc(pSub)){ + daslog_error_v("Child '%s' of %s loaded but is not a stream source", sPick, DasNode_srcUrl(pNode)); + return NULL; + } + return pSub; +} diff --git a/das3/source.h b/das3/source.h new file mode 100644 index 0000000..db508d6 --- /dev/null +++ b/das3/source.h @@ -0,0 +1,635 @@ +/* Copyright (C) 2026 Chris Piker + * + * This file is part of das2C, the Core Das2 C Library. + * + * Das2C is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License version 2.1 as published + * by the Free Software Foundation. + * + * Das2C is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for + * more details. + * + * You should have received a copy of the GNU Lesser General Public License + * version 2.1 along with das2C; if not, see . + */ + +/** @file source.h Data sources: what they offer and how to ask for it */ + +#ifndef _das_source_h_ +#define _das_source_h_ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** @addtogroup catalog + * @{ + */ + +/** @defgroup source Data sources + * + * A DasSrc is a standalone description of one place data comes from, + * built from a DasNode or directly from JSON text. It holds a copy of + * that document, so the catalog tree it came from may be freed or + * evicted and the source keeps working. Once built it is read only and + * may be shared between threads. + * + * Three things fall out of the description: + * + * - **Properties.** The `interface` section of a source node is a + * tree of sections, groups and properties. A DasSrcProp is one leaf: + * the minimum time, the units of the frequency axis, whether the + * L-shell variable is output. Every property has a value; some also + * have a DasSrcCtrl saying how a client may change it. The fixed ones + * are not noise, they tell a client what a stream will look like + * before any of it is downloaded. + * - **Groups.** A DasSrcGroup is the box the properties sit in: a + * coordinate, a data variable, an output format, or the one anonymous + * box of server options. Forms are drawn group by group. + * - **Requisitions.** A DasSrc makes DasReq objects, blank or seeded + * from a worked example, validates them, and turns them into + * something a transport can act on. For a stream source that is a + * URL. + * + * A DasSrc resolves a requisition to a URL but never fetches from one. + * Nothing in it opens a socket, so the same code serves a command line + * tool, a browser build, or a GUI toolkit that provides the network + * functions. Facilities such as das_http_getBody() and DasIO act on the + * URL afterwards. + * + * The common path, through a DasCat: + * + * @code + * DasSrc* pSrc = DasCat_newSrc(pCat, "uiowa/juno/wav/survey"); + * DasReq* pReq = DasSrc_newReq(pSrc, 0); // seeded from example 0 + * DasReq_set(pReq, "time.res", "30"); // then adjusted + * + * char sUrl[2048]; + * if(DasStreamSrc_url(pSrc, pReq, 0, sUrl, sizeof(sUrl)) > 0) + * das_http_getBody(sUrl, ...); // or fetch(), or QNAM + * + * del_DasReq(pReq); + * del_DasSrc(pSrc); + * @endcode + * + * @anchor src_keys + * @par Keys + * + * Every property has a dotted key built from its section, group and + * property names as the node spells them: `coords.time.min`, + * `data.lshell.enabled`, `formats.csv.delimiter`, `options.ascii`. + * Options have no group, so their keys have two parts. Keys are how a + * requisition names a property, and these shorter spellings are accepted + * as well, so long as the result is unambiguous within one source: + * + * - Section names in singular or plural form, `coord` or `coords`, + * `format` or `formats`, `option` or `options`, and `fmt` or `opt`. + * - The group and property alone, `time.min`, when only one section + * has that group. + * - A group alone, `lshell` or `data.lshell`, meaning the group's + * `enabled` toggle. + * - An option alone, `ascii`, when no group has that name. + * - A bare property name, `serial`, when exactly one group has it. + * - The property names `min`, `max`, `res` and `inter` may be written + * out as `minimum`, `maximum`, `resolution` and `interval`, and + * `enable` as `enabled`. Nodes from 2018 spell them the long way, + * later nodes the short way, and users may type either. + * + * Matching is case insensitive. A spelling that fits more than one + * property is rejected with a message naming the candidates, never + * guessed. + * + * A coordinate name alone, `time`, takes range values: + * + * - `time=2021-02-21,2021-02-22` sets the minimum and maximum, and a + * third item sets the resolution or the interval, whichever the + * coordinate has. Empty items are skipped, so `time=,2021-02-22` + * sets only the maximum. + * - `time=2021-02-21` with no comma means the span that time denotes: + * the whole day; `time=2021-02-21T14` the hour from 14:00 to 15:00; + * `time=2021` the year. See dt_parsespan() for the rule. + * + * In a well formed source, the group names under `coords` are the physical + * dimension names of the emitted stream, @see DasDim. A client that has + * plotted a stream and needs more data along `time` can look up that + * group and issue a new requisition from the plot's axis range, which + * is how zoom and pan become server requests without any source specific + * code in the client. + * + * @par Drawing a form + * + * @code + * for(int sec = 0; sec < das_src_sec_size; ++sec){ + * for(size_t g = 0; g < DasSrc_groupCount(pSrc, sec); ++g){ + * const DasSrcGroup* pG = DasSrc_group(pSrc, sec, g); + * const DasSrcProp* pOn = DasSrcGroup_enable(pG); // the gate, or NULL + * + * for(size_t p = 0; p < DasSrcGroup_propCount(pG); ++p){ + * const DasSrcProp* pProp = DasSrcGroup_prop(pG, p); + * const DasSrcCtrl* pCtrl = DasSrcProp_ctrl(pProp); + * if(pCtrl == NULL) continue; // nothing to set here + * + * const char* sNow = DasReq_value(pReq, DasSrcProp_key(pProp)); + * draw_control(DasSrcProp_type(pProp), pCtrl, sNow ? sNow : DasSrcProp_value(pProp)); + * } + * } + * } + * @endcode + * + * The verdicts to mark boxes with come from DasSrc_check(), read back by + * entry through DasReq_verdict() and DasReq_why(). + * + * @par Two node layouts + * + * Source nodes from 2018 and from current servers differ in spelling and + * in where examples live. Both are read and nothing here depends on + * which one a source uses: examples from either come out as requisitions + * in interface terms. + */ + +/** The parts of a source interface, in presentation order. + * `das_src_sec_size` is the number of real sections, handy for sizing + * per section arrays. */ +typedef enum das_src_sec_enum { + das_src_sec_coord = 0, /**< "coords": time, frequency, L-shell ... */ + das_src_sec_data, /**< "data": the variables a source can emit */ + das_src_sec_format, /**< "formats": output serializations and their knobs */ + das_src_sec_option, /**< "options": server specific switches, one group */ + das_src_sec_size +} das_src_sec_e; + +/** The name of a section as it appears in keys, "coords", "data", + * "formats" or "options". "" for das_src_sec_size. */ +DAS_API const char* das_src_sec_name(das_src_sec_e sec); + +/** Kinds of source. Only stream sources are implemented today. */ +typedef enum das_src_kind_enum { + das_src_kind_inv = 0, + das_src_kind_stream, /**< node type HttpStreamSrc, see DasStreamSrc_url() */ + das_src_kind_file /**< node type FileTimeAgg, reserved */ +} das_src_kind_e; + +/** The kind of value a property holds. + * + * Nodes name types with a handful of spellings that have drifted over + * the years. They are folded into this set when a source is built. A + * type the library does not recognize is reported as text and the + * node's spelling is still available via DasSrcProp_typeName(). + */ +typedef enum das_srcprop_type_enum { + das_srcprop_text = 0, /**< any string, including unrecognized types */ + das_srcprop_bool, /**< true or false, see DasSrcCtrl_check() for spellings */ + das_srcprop_int, /**< an integer in decimal */ + das_srcprop_real, /**< a real number */ + das_srcprop_time, /**< a UTC time, ISO 8601 */ + das_srcprop_enum /**< one of a fixed list, see DasSrcCtrl_enumCount() */ +} das_srcprop_type_e; + +/** A data source description. See the group description. */ +typedef struct das_src DasSrc; + +/** A box on the form. Owned by its source. */ +typedef struct das_src_group DasSrcGroup; + +/** One property of a source, fixed or adjustable. Owned by its source. */ +typedef struct das_src_prop DasSrcProp; + +/** How an adjustable property may be changed. Owned by its property. */ +typedef struct das_src_ctrl DasSrcCtrl; + +/* ************************************************************************* */ +/* Making and identifying sources */ + +/** Build a DasSrc from a DasNode. + * + * The node's document is copied, so the node and its tree may be freed + * afterwards. Usually reached through DasCat_newSrc(), which also + * picks the right child of a container node. + * + * @param pNode A node of type HttpStreamSrc. Borrowed. + * @returns A new source, to be freed with del_DasSrc(), or NULL after + * logging why the node does not describe a source. A source with + * a thin or broken interface still builds, so that it can at + * least be listed; DasStreamSrc_url() is where missing protocol + * pieces are reported. + */ +DAS_API DasSrc* new_DasSrc_node(const DasNode* pNode); + +/** Build a DasSrc from the JSON text of a node, with no DasNode involved. + * + * For programs that fetch documents themselves, a browser build in + * particular, and for anything else that can produce a source document. + * + * @param sDoc The JSON text, need not be NUL terminated. + * @param uLen The number of bytes in sDoc. + * @param sPathUri The catalog URI of the node, shorthands accepted. It + * is the source's identity: requisitions are stamped with it and + * later checked against it, see DasSrc_check(). NULL or empty + * names the source after sUrl under D2URI_LOCAL, or as + * "local:/memory" when there is no URL either. + * @param sUrl Where the text came from, recorded as the source's URL and + * used to resolve relative links. May be NULL. + * @returns A new source, or NULL after logging a parse problem or a + * document that is not a source node. + */ +DAS_API DasSrc* new_DasSrc_json( + const char* sDoc, size_t uLen, const char* sPathUri, const char* sUrl +); + +DAS_API void del_DasSrc(DasSrc* pThis); + +/** @memberof DasSrc */ +DAS_API das_src_kind_e DasSrc_kind(const DasSrc* pThis); + +/** The catalog URI, the source's identity. Never empty: a source loaded + * by location with no URI given is named under D2URI_LOCAL. + * @memberof DasSrc */ +DAS_API const char* DasSrc_path(const DasSrc* pThis); + +/** Where the source document was loaded from, "" if unknown. @memberof DasSrc */ +DAS_API const char* DasSrc_url(const DasSrc* pThis); + +/** The node's short name, label, or "" . @memberof DasSrc */ +DAS_API const char* DasSrc_name(const DasSrc* pThis); + +/** The node's title, or "". @memberof DasSrc */ +DAS_API const char* DasSrc_title(const DasSrc* pThis); + +/** The node format version, "0.7" and so on, or "". @memberof DasSrc */ +DAS_API const char* DasSrc_version(const DasSrc* pThis); + +/** True when the protocol section says requests are authenticated. + * Servers may still ask when this is false; it is advice for the form, + * not a promise. @memberof DasSrc */ +DAS_API bool DasSrc_needsAuth(const DasSrc* pThis); + +/** The number of listed contacts for the data. @memberof DasSrc */ +DAS_API size_t DasSrc_contactCount(const DasSrc* pThis); + +/** Contact number uIdx. + * @param psEmail Receives the address, or "" if none listed. May be NULL. + * @returns The name, or NULL past the end. @memberof DasSrc */ +DAS_API const char* DasSrc_contact(const DasSrc* pThis, size_t uIdx, const char** psEmail); + +/** The source document, or an element of it by DasJdo_get() path, for + * anything the functions here do not model. NULL when the path finds + * nothing. @memberof DasSrc */ +DAS_API const DasJdo* DasSrc_jdo(const DasSrc* pThis, const char* sFragment); + +/* ************************************************************************* */ +/* Groups */ + +/** The number of groups in a section, 0 for a section the source does + * not have. Coordinates list `time` first and the rest in node order; + * the options section has one group when it has anything. + * @memberof DasSrc */ +DAS_API size_t DasSrc_groupCount(const DasSrc* pThis, das_src_sec_e sec); + +/** Group number uIdx of a section, or NULL past the end. @memberof DasSrc */ +DAS_API const DasSrcGroup* DasSrc_group(const DasSrc* pThis, das_src_sec_e sec, size_t uIdx); + +/** The group of a section with a given name, case insensitively, or + * NULL. Pass "" for the options group. @memberof DasSrc */ +DAS_API const DasSrcGroup* DasSrc_findGroup( + const DasSrc* pThis, das_src_sec_e sec, const char* sName +); + +/** @memberof DasSrcGroup */ +DAS_API das_src_sec_e DasSrcGroup_section(const DasSrcGroup* pThis); + +/** The name as the node spells it, "time", "lshell", "csv", or "" for + * the options group. @memberof DasSrcGroup */ +DAS_API const char* DasSrcGroup_name(const DasSrcGroup* pThis); + +/** A short caption: the node's label, else its name. @memberof DasSrcGroup */ +DAS_API const char* DasSrcGroup_label(const DasSrcGroup* pThis); + +/** A longer description, or "". @memberof DasSrcGroup */ +DAS_API const char* DasSrcGroup_title(const DasSrcGroup* pThis); + +/** The property that switches the whole group on and off, or NULL. + * + * Data variables usually have one, formats always do, coordinates and + * options never. A form draws it first and greys the rest of the group + * when it is off. @memberof DasSrcGroup */ +DAS_API const DasSrcProp* DasSrcGroup_enable(const DasSrcGroup* pThis); + +/** The name of the exclusive set this group's enable toggle belongs to, + * or NULL. Formats are in one, "format": exactly one is on at a time, + * so they are radio buttons rather than check boxes. @memberof DasSrcGroup */ +DAS_API const char* DasSrcGroup_xorGroup(const DasSrcGroup* pThis); + +/** The number of MIME types a format group may produce, 0 for other + * groups. @memberof DasSrcGroup */ +DAS_API size_t DasSrcGroup_mimeCount(const DasSrcGroup* pThis); + +/** MIME type number uIdx of a format group, or NULL. The first is the + * usual one. @memberof DasSrcGroup */ +DAS_API const char* DasSrcGroup_mime(const DasSrcGroup* pThis, size_t uIdx); + +/** @memberof DasSrcGroup */ +DAS_API size_t DasSrcGroup_propCount(const DasSrcGroup* pThis); + +/** Property number uIdx of the group, in node order, or NULL. + * @memberof DasSrcGroup */ +DAS_API const DasSrcProp* DasSrcGroup_prop(const DasSrcGroup* pThis, size_t uIdx); + +/** The property of a group with a given name, accepting the long and + * short spellings listed under @ref src_keys, or NULL. @memberof DasSrcGroup */ +DAS_API const DasSrcProp* DasSrcGroup_findProp(const DasSrcGroup* pThis, const char* sName); + +/** How many of the group's properties have controls. 0 means a fixed + * group, which a form may skip and a verbose listing may grey out. + * @memberof DasSrcGroup */ +DAS_API size_t DasSrcGroup_ctrlCount(const DasSrcGroup* pThis); + +/* ************************************************************************* */ +/* Properties */ + +/** The number of properties over all sections. @memberof DasSrc */ +DAS_API size_t DasSrc_propCount(const DasSrc* pThis); + +/** Property number uIdx in presentation order: sections as in + * das_src_sec_e, groups and properties within them in group order, so + * a group's properties are always adjacent. @memberof DasSrc */ +DAS_API const DasSrcProp* DasSrc_prop(const DasSrc* pThis, size_t uIdx); + +/** Find the one property a key names. + * + * Any spelling from @ref src_keys is accepted. Fixed properties are + * found too, so a caller that means to set one still has to look at + * DasSrcProp_ctrl(). A key that fits nothing or more than one property + * returns NULL; the reason is not logged, since this is also how a + * command line tool asks "is this a key", and sWhy says it instead. + * + * @param sKey The key as typed. + * @param sWhy If not NULL, receives "" on success or a message naming + * the problem and, for an ambiguous key, the candidates. + * @param uWhy The size of sWhy. + * @returns The property, or NULL. + * @memberof DasSrc + */ +DAS_API const DasSrcProp* DasSrc_findProp( + const DasSrc* pThis, const char* sKey, char* sWhy, size_t uWhy +); + +/** @memberof DasSrcProp */ +DAS_API das_src_sec_e DasSrcProp_section(const DasSrcProp* pThis); + +/** The group this property belongs to. @memberof DasSrcProp */ +DAS_API const DasSrcGroup* DasSrcProp_group(const DasSrcProp* pThis); + +/** The property name as the node spells it, "min" or "minimum", + * "enabled", "delimiter". Use DasSrcProp_key() rather than rebuilding + * keys from this. @memberof DasSrcProp */ +DAS_API const char* DasSrcProp_name(const DasSrcProp* pThis); + +/** The full dotted key, see @ref src_keys. Unique within a source. + * @memberof DasSrcProp */ +DAS_API const char* DasSrcProp_key(const DasSrcProp* pThis); + +/** A short human readable title. + * + * The node's title is returned when it has one. Coordinate range + * properties and enable toggles usually have none, since their meaning + * is conventional, so one is composed in sBuf from the group's label, + * "Minimum Time value to output", "Output as CSV". + * + * @param sBuf Space for a composed title, 64 bytes covers the composed + * forms. + * @param uLen The size of sBuf. + * @returns A pointer to the title, inside the source or in sBuf. Never + * NULL, but may be "" for an option with no title at all. + * @memberof DasSrcProp + */ +DAS_API const char* DasSrcProp_title(const DasSrcProp* pThis, char* sBuf, size_t uLen); + +/** The units the value is in, as the node states them, "UTC" for a time + * with no stated units, or "". Display only, values are not checked + * against units. @memberof DasSrcProp */ +DAS_API const char* DasSrcProp_units(const DasSrcProp* pThis); + +/** @memberof DasSrcProp */ +DAS_API das_srcprop_type_e DasSrcProp_type(const DasSrcProp* pThis); + +/** The node's spelling of the type, "isotime", "boolean", or "" + * when it gave none and the type was inferred. @memberof DasSrcProp */ +DAS_API const char* DasSrcProp_typeName(const DasSrcProp* pThis); + +/** The property's value as the node spells it, numbers in the node's + * notation. + * + * For a fixed property this is the fact: the units of an axis, the + * valid range of a coordinate. For an adjustable one it is the value in + * effect when a requisition says nothing, the default. NULL when the + * node gives null or nothing, which for a required control means the + * client must supply one. @memberof DasSrcProp */ +DAS_API const char* DasSrcProp_value(const DasSrcProp* pThis); + +/** The control for an adjustable property, NULL for a fixed one. + * @memberof DasSrcProp */ +DAS_API const DasSrcCtrl* DasSrcProp_ctrl(const DasSrcProp* pThis); + +/* ************************************************************************* */ +/* Controls */ + +/** The property this control adjusts. @memberof DasSrcCtrl */ +DAS_API const DasSrcProp* DasSrcCtrl_prop(const DasSrcCtrl* pThis); + +/** True when a requisition must supply a value or the server will refuse + * the request. A required control whose property has a value is + * satisfied by that default. @memberof DasSrcCtrl */ +DAS_API bool DasSrcCtrl_required(const DasSrcCtrl* pThis); + +/** The name of the protocol parameter a value travels in, "start_time" + * or "read.time.min". Several controls may share one, toggles that pack + * into a flag set in particular. For messages and for people composing + * requests by hand. @memberof DasSrcCtrl */ +DAS_API const char* DasSrcCtrl_param(const DasSrcCtrl* pThis); + +/** The number of allowed values of an enumerated control, 0 otherwise. + * @memberof DasSrcCtrl */ +DAS_API size_t DasSrcCtrl_enumCount(const DasSrcCtrl* pThis); + +/** Allowed value number uIdx as the node spells it, or NULL. Checks + * accept these in any letter case; what is sent is the node's spelling. + * @memberof DasSrcCtrl */ +DAS_API const char* DasSrcCtrl_enum(const DasSrcCtrl* pThis, size_t uIdx); + +/** The stated range of a numeric or time control. + * + * Nodes put ranges in several places: on the set block, on the protocol + * parameter, or for coordinate `min` and `max` on the coordinate itself + * as its valid range. All three are consulted, in that order. The high + * end of a time range is often the word "now". + * + * @param psLo Receives the low end, as the node spells it. May be NULL. + * @param psHi Receives the high end. May be NULL. + * @returns true if a range was stated, false otherwise with the outputs + * untouched. + * @memberof DasSrcCtrl + */ +DAS_API bool DasSrcCtrl_range(const DasSrcCtrl* pThis, const char** psLo, const char** psHi); + +/** Decide if a string is an acceptable value for this control. + * + * Booleans accept true, false, yes, no, on, off, 1 and 0 in any letter + * case. Enumerations accept any listed value, case insensitively. + * Integers, reals and times are checked for form only, not against the + * stated range, because servers are the authority on what they hold and + * ranges in nodes are routinely out of date. Times also accept the word + * "now". Text is always acceptable. + * + * DasSrc_check() calls this for every entry, so applications rarely + * need to, but a settings panel may want to validate as the user types. + * + * @param sWhy If not NULL, receives "" or a message naming the key, the + * value and, for an enumeration, the choices. + * @param uWhy The size of sWhy. + * @returns DAS_OKAY or DASERR_SRC. Nothing is logged. + * @memberof DasSrcCtrl + */ +DAS_API DasErrCode DasSrcCtrl_check( + const DasSrcCtrl* pThis, const char* sVal, char* sWhy, size_t uWhy +); + +/* ************************************************************************* */ +/* Requisitions */ + +/** The number of worked examples a source provides, possibly 0. + * @memberof DasSrc */ +DAS_API size_t DasSrc_exampleCount(const DasSrc* pThis); + +/** What example number uIdx shows, from its label, title or name, "" if + * it has none, NULL past the end. @memberof DasSrc */ +DAS_API const char* DasSrc_exampleLabel(const DasSrc* pThis, size_t uIdx); + +/** The argument to DasSrc_newReq() for a requisition with nothing filled in */ +#define DASSRC_BLANK_REQ -1 + +/** Make a DasReq for this source. + * + * With DASSRC_BLANK_REQ the requisition lists every required key with an + * empty value, so a form knows what must be filled and a check on it + * fails usefully. With an example number it carries that example's + * settings instead, in interface terms whatever the node layout, and the + * example's label. Either way it is stamped for this source. + * + * Nothing stops a client from making a requisition with new_DasReq() + * and never involving the source until it checks; this is the + * convenience that knows what the source wants. + * + * @returns A new requisition, to be freed with del_DasReq(), or NULL + * after logging that there is no such example. + * @memberof DasSrc + */ +DAS_API DasReq* DasSrc_newReq(const DasSrc* pThis, int nExample); + +/** Stamp a requisition as belonging to this source, see DasReq_setSrc(). + * @memberof DasSrc */ +DAS_API void DasSrc_stamp(const DasSrc* pThis, DasReq* pReq); + +/** Validate a request against this source. + * + * Every entry gets a verdict and the bad ones a reason, see DasReq_why(). + * In order: + * + * 1. A DasReq stamped with another catalog URI is refused outright + * and nothing is validated. Only the URI is compared, so a mirror + * or a local mount of the same node validates the same requisitions. + * An unstamped one is validated by whoever holds it. + * 2. Each key is resolved as DasSrc_findProp() would. No match, + * more than one match, or a fixed property is bad. A coordinate + * group name with range values, see @ref src_keys, resolves to + * derived entries for the minimum, maximum and step; the client's + * entry stays so that a form's one box stays one box. + * 3. Each value is checked with DasSrcCtrl_check(). An empty value + * is bad for a required control with no default, and otherwise + * means "not set". + * 4. Every required control not covered by an entry or a default gets + * a derived entry with an empty value and a bad verdict, so a form + * can mark a box the user never touched. + * + * Derived entries from an earlier check are dropped first, so checking + * is idempotent. Nothing is logged for bad entries, the verdicts carry + * the text; DasStreamSrc_url() logs the first one when it refuses to + * build. + * + * @param puBad If not NULL, receives the number of bad entries. + * @returns DAS_OKAY when every entry is good, DASERR_SRC when any is bad + * or the stamp does not match, the latter after logging. + * @memberof DasSrc + */ +DAS_API DasErrCode DasSrc_check(const DasSrc* pThis, DasReq* pReq, size_t* puBad); + +/** The MIME type a requisition is expected to produce. + * + * Taken from the enabled format group after the requisition's choices, + * or the node's default when it states one. Where a group lists several + * types the first is returned; the server's Content-Type header is the + * final word. Tells a client whether to parse the body as a das stream, + * hand it to a table viewer, or write it to disk. + * + * @returns The type, or "" when the source does not say. + * @memberof DasSrc + */ +DAS_API const char* DasSrc_mime(const DasSrc* pThis, const DasReq* pReq); + +/* ************************************************************************* */ +/* Stream sources */ + +/** The number of base URLs a stream source may be reached at, usually + * 1. More means mirrors, tried in order by a careful client. 0 for a + * source that is not a stream source or has no protocol section. + * @memberof DasSrc */ +DAS_API size_t DasStreamSrc_urlCount(const DasSrc* pThis); + +/** Base URL number uIdx, before any parameters, or NULL. Old nodes + * carry a query string in it already; DasStreamSrc_url() appends either + * way. @memberof DasSrc */ +DAS_API const char* DasStreamSrc_baseUrl(const DasSrc* pThis, size_t uIdx); + +/** The request URL for a requisition. + * + * Checks the requisition, then translates each good entry to protocol + * parameters through its control's set block: a plain value, a fixed + * protocol value, or a flag packed into a parameter shared with others + * of its set. A value equal to the default is not sent unless the + * control is required; required controls with defaults are sent even + * when the requisition is silent. Raw parameters go on last and + * override. Finally every protocol parameter the node marks required + * must be present. Values are percent encoded, see das_url_encode(). + * + * Works like snprintf: the return value is the length the whole URL + * needs, and the buffer holds as much as fits, NUL terminated. + * + * @param pReq The requisition. Its verdicts are updated as by + * DasSrc_check(). + * @param uBase Which base URL, see DasStreamSrc_urlCount(). 0 unless an + * earlier attempt failed to connect. + * @param sBuf Receives the URL. 2048 bytes is enough for any node seen + * so far, 4096 is safe. + * @param uLen The size of sBuf. + * @returns The length of the URL, or 0 after logging why none could be + * made: a bad entry, a mismatched stamp, a missing required + * parameter no key supplies, a bad base index, or a source with + * no base URLs. A URL is never empty, so 0 is unambiguous. + * @memberof DasSrc + */ +DAS_API size_t DasStreamSrc_url( + const DasSrc* pThis, DasReq* pReq, size_t uBase, char* sBuf, size_t uLen +); + +/** @} */ +/** @} */ + +#ifdef __cplusplus +} +#endif + +#endif /* _das_source_h_ */ diff --git a/das3/src_priv.h b/das3/src_priv.h new file mode 100644 index 0000000..c0bebed --- /dev/null +++ b/das3/src_priv.h @@ -0,0 +1,163 @@ +/* Copyright (C) 2026 Chris Piker + * + * This file is part of das2C, the Core Das2 C Library. + * + * Das2C is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License version 2.1 as published + * by the Free Software Foundation. + * + * Das2C is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for + * more details. + * + * You should have received a copy of the GNU Lesser General Public License + * version 2.1 along with das2C; if not, see . + */ + +/* Source internals. Not installed. + * + * Everything in source.c sees these; catalog.c includes this header only + * for the default source rule. Applications see the accessors in + * source.h and nothing of the layout, so it can change after the tag. + * + * Ownership in one sentence: the source owns its document copy, its + * groups, its properties and their controls, and the example + * requisitions; everything that points into the document is borrowed + * from it and dies with it. + */ + +#ifndef _das_src_priv_h_ +#define _das_src_priv_h_ + +#include "source.h" +#include "json.h" +#include "req_priv.h" + +/* A control. Borrowed pointers point into the source's document. JSON + * number text is not NUL terminated, hence the owned copies of enum and + * range strings. */ +struct das_src_ctrl { + DasSrcProp* pProp; /* back pointer, the owner */ + const DasJdo* pSet; /* the set dict: param, flag, pval, enum, range, value */ + const DasJdo* pParam; /* the protocol parameter entry for sParam, or + NULL if the node does not describe it */ + const char* sParam; /* borrowed from pSet, never NULL */ + bool bRequired; /* set/required or the parameter's required */ + char** aEnum; /* owned, enum values in node order */ + size_t uEnum; + char* sLo; /* owned, range ends, NULL when none stated */ + char* sHi; +}; + +struct das_src_prop { + das_src_sec_e sec; + DasSrcGroup* pGroup; /* back pointer */ + const char* sName; /* borrowed dict key, the node's spelling */ + char* sKey; /* owned, "coords.time.min" */ + const DasJdo* pJdo; /* the property dict: value, set, type, label... */ + das_srcprop_type_e type; + const char* sTypeName; /* borrowed, "" when inferred */ + char* sValue; /* owned scalar copy, NULL when null or absent */ + DasSrcCtrl* pCtrl; /* owned, NULL for a fixed property */ +}; + +/* Properties of a group are adjacent in the source's aProp array, so a + * group is an index range. */ +struct das_src_group { + const DasSrc* pSrc; /* back pointer, for reaching the properties */ + das_src_sec_e sec; + const char* sName; /* borrowed dict key, "" for options */ + const DasJdo* pJdo; /* the coordinate, variable, format or options dict */ + size_t uFirst; /* into DasSrc.aProp */ + size_t uCount; + int nEnable; /* index of the "enabled" prop, -1 if none */ + const char* sXor; /* borrowed, enable prop's xorGroup, or NULL */ + const char** aMime; /* owned array of borrowed strings, formats only */ + size_t uMime; +}; + +struct das_src { + das_src_kind_e kind; + char* sPath; /* owned, "" when none */ + char* sUrl; /* owned, "" when none */ + + DasJdo* pDoc; /* owned, the parsed copy of the node document */ + const DasJdo* pIface; /* "interface", may be NULL */ + bool bLegacy; /* the 2018 layout, see _DasSrc_isLegacy() */ + + DasSrcGroup* aGroup; /* owned, in section then presentation order */ + size_t uGroup; + size_t aSecFirst[das_src_sec_size]; /* first group of each section */ + size_t aSecCount[das_src_sec_size]; + + DasSrcProp* aProp; /* owned, grouped as the groups say */ + size_t uProp; + DasSrcCtrl* aCtrl; /* owned, one per settable property, in + property order; props point into it */ + size_t uCtrl; + + DasReq** aExample; /* owned, one per example, built at load */ + size_t uExample; +}; + +/* The stream source. Base first so a DasStreamSrc* is a DasSrc*; the cast + * back is made only after kind says das_src_kind_stream. */ +typedef struct das_stream_src { + DasSrc base; + const DasJdo* pProto; /* "protocol", NULL makes the source unqueryable */ + const DasJdo* pParams; /* protocol "httpParams" or "http_params" */ + const DasJdo* pBaseUrls; /* protocol "baseUrls" or "base_urls" array */ +} DasStreamSrc; + +/* Key grammar helpers, see "Keys" in source.h. They live next to the + * parser so that the spellings accepted on input and the spellings + * written into keys cannot drift apart. */ + +/* Fold a section name as a user may type it to a section. Case + * insensitive. Returns false if the word is not a section name. */ +bool _das_src_secByName(const char* sName, das_src_sec_e* pSec); + +/* The short spelling of a property name: "minimum" -> "min", "enable" -> + * "enabled". Names with no alias are returned as given. */ +const char* _das_src_propName(const char* sProp); + +/* True if two property names are the same after _das_src_propName() and + * case folding. */ +bool _das_src_propEq(const char* sA, const char* sB); + +/* Split a dotted key into at most three parts. + * + * @param aParts Receives the parts, each NUL terminated. + * @returns The number of parts, 1 to 3, or 0 if the key is empty, has an + * empty part, has more than three parts, or has a part that does + * not fit. */ +int _das_src_splitKey(const char* sKey, char aParts[3][DAS_MAX_ID_BUFSZ]); + +/* Find properties matching a key, the engine behind DasSrc_findProp() + * and DasSrc_check(). + * + * Tries the spellings from most to least specific: SEC.GROUP.PROP; + * SEC.PROP for options or SEC.GROUP for a toggle; GROUP.PROP in any + * section; a lone option name or group toggle; and last a bare property + * name. The first level that matches anything decides, so a lone name + * that is both an option and a property reports the option. + * + * @param aOut Receives up to nMax matches. May be NULL with nMax 0 to + * count only. + * @returns The number of matches, which may exceed nMax. 0 is no such + * key, 1 a hit, more is ambiguous and aOut holds the candidates. */ +int _DasSrc_match(const DasSrc* pThis, const char* sKey, const DasSrcProp** aOut, int nMax); + +/* If sKey, with or without a section prefix, names a coordinate group, + * return that group, else NULL. How a check recognizes the range forms. */ +const DasSrcGroup* _DasSrc_coordGroup(const DasSrc* pThis, const char* sKey); + +/* The default source under a container node: the node itself if it is a + * stream source, else the child named "flex", else the first stream + * source child. Loads the child through the node's own tree, so + * credentials and cache settings follow. Returns NULL after logging + * when there is none. Lives in source.c, used by DasCat_newSrc(). */ +DasNode* _DasSrc_defNode(DasNode* pNode); + +#endif /* _das_src_priv_h_ */ diff --git a/das3/time.c b/das3/time.c index e1396f7..02e1778 100644 --- a/das3/time.c +++ b/das3/time.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2024 Chris Piker +/* Copyright (C) 2015-2026 Chris Piker * Copyright (C) 1993-1998 Larry Granroth * * This file used to be named parsetime.c. It is part of das2C, the Core @@ -389,6 +389,106 @@ bool dt_parsetime(das_time* dt, const char* sTime) else return false; } +/* The span rule needs to know which field was the last one written, and + * parsetime() does not say, so the string is scanned for its shape + * first and parsetime() is trusted for the values. Shapes handled: + * Y, Y-M, Y-M-D, Y-DOY, each optionally followed by T or space and + * h, h:m, h:m:s, with an optional trailing Z. Anything else, including + * fractional seconds, is refused. */ +bool dt_parsespan(const char* sTime, das_time* pBeg, das_time* pEnd) +{ + if(sTime == NULL) return false; + char sBuf[64]; + strncpy(sBuf, sTime, sizeof(sBuf) - 1); + sBuf[sizeof(sBuf) - 1] = '\0'; + + /* trim blanks and a trailing Z */ + char* pS = sBuf; + while(isspace((unsigned char)*pS)) ++pS; + size_t uLen = strlen(pS); + while((uLen > 0) && isspace((unsigned char)pS[uLen - 1])) pS[--uLen] = '\0'; + if((uLen > 0) && ((pS[uLen - 1] == 'Z')||(pS[uLen - 1] == 'z'))) pS[--uLen] = '\0'; + if(uLen == 0) return false; + + /* the date and the time of day */ + char* pTod = strchr(pS, 'T'); + if(pTod == NULL) pTod = strchr(pS, ' '); + if(pTod != NULL){ *pTod = '\0'; ++pTod; if(*pTod == '\0') return false; } + + /* count fields and insist on digits */ + int nDate = 0, aDateLen[3] = {0, 0, 0}; + for(const char* p = pS; ; ++p){ + if((*p == '-')||(*p == '\0')){ + if(nDate == 3) return false; + if(aDateLen[nDate] == 0) return false; + ++nDate; + if(*p == '\0') break; + } + else if(isdigit((unsigned char)*p)) aDateLen[nDate] += 1; + else return false; + } + int nTod = 0; + if(pTod != NULL){ + int nLen = 0; + for(const char* p = pTod; ; ++p){ + if((*p == ':')||(*p == '\0')){ + if((nTod == 3)||(nLen == 0)) return false; + ++nTod; nLen = 0; + if(*p == '\0') break; + } + else if(isdigit((unsigned char)*p)) ++nLen; + else return false; /* a '.' lands here: fractions have no span */ + } + } + + /* which unit steps: 0 year, 1 month, 2 day, 3 hour, 4 minute, 5 second */ + int nUnit; + bool bDoy = (nDate == 2) && (aDateLen[1] == 3); + if(nTod > 0){ + if((nDate < 3) && !bDoy) return false; /* a time of day needs a whole date */ + nUnit = 2 + nTod; + } + else{ + nUnit = bDoy ? 2 : nDate - 1; + } + + das_time dt; + dt_null(&dt); + if(nUnit == 0){ + /* parsetime() refuses a bare year */ + dt.year = atoi(pS); + dt.month = 1; dt.mday = 1; + if(dt.year <= 0) return false; + } + else if(nUnit == 1){ + dt.year = atoi(pS); + dt.month = atoi(strchr(pS, '-') + 1); + dt.mday = 1; + if((dt.year <= 0)||(dt.month < 1)||(dt.month > 12)) return false; + } + else{ + if(pTod != NULL) *(pTod - 1) = 'T'; /* put the string back together */ + if(!dt_parsetime(&dt, pS)) return false; + } + dt_tnorm(&dt); + + das_time dtEnd; + dt_copy(&dtEnd, &dt); + switch(nUnit){ + case 0: dtEnd.year += 1; break; + case 1: dtEnd.month += 1; break; + case 2: dtEnd.mday += 1; break; + case 3: dtEnd.hour += 1; break; + case 4: dtEnd.minute += 1; break; + default: dtEnd.second += 1.0; break; + } + dt_tnorm(&dtEnd); + + if(pBeg) dt_copy(pBeg, &dt); + if(pEnd) dt_copy(pEnd, &dtEnd); + return true; +} + /* ************************************************************************* */ bool dt_now(das_time* pDt) { diff --git a/das3/time.h b/das3/time.h index d227c9f..768886e 100644 --- a/das3/time.h +++ b/das3/time.h @@ -1,5 +1,5 @@ -/* Copyright (C) 1997-2017 Larry Granroth - * Chris Piker +/* Copyright (C) 1997 Larry Granroth + * 2012-2026 Chris Piker * * This file is part of das2C, the Core Das2 C Library. * @@ -127,6 +127,26 @@ DAS_API void dt_null(das_time* pDt); */ DAS_API bool dt_parsetime(das_time* pDt, const char* sTime); +/** Parse a time string as the span it denotes. + * + * A time written to limited precision names an interval, not an instant: + * "2026-01-02" is the whole of January 2nd, "2026-01-02T14" the hour from + * 14:00 to 15:00, "2026" the year. This reads the string the way + * dt_parsetime() does and also sets pEnd to the start of the next unit + * of the least significant field given. Day of year forms, "2026-002", + * step the day. A string with a fraction of a second, "T14:30:15.5", + * denotes no natural span and is refused, as is anything dt_parsetime() + * refuses. + * + * @param sTime An ISO 8601 date or date and time, year through whole + * seconds, with or without a trailing Z. + * @param pBeg Receives the start of the span. May be NULL. + * @param pEnd Receives the end of the span, exclusive. May be NULL. + * @returns true if the string was understood, false otherwise with both + * outputs untouched. + */ +DAS_API bool dt_parsespan(const char* sTime, das_time* pBeg, das_time* pEnd); + /** Initialize a das_time to the current UTC time. * diff --git a/test/TestCat.c b/test/TestCat.c index f167c53..7cca3d5 100644 --- a/test/TestCat.c +++ b/test/TestCat.c @@ -37,6 +37,7 @@ #include #include #include +#include #include #include @@ -49,6 +50,17 @@ fixtures are local files, which are never cached. */ #define TEST_USERDIR "test_das_userdir_never_created" +/* The local: name a relative test file gets: the working directory is + * part of it, see D2URI_LOCAL. Static buffer, one call per CHECK. */ +static const char* localUri(const char* sFile) +{ + static char sBuf[1024]; + char sCwd[512] = {'\0'}; + if(getcwd(sCwd, sizeof(sCwd) - 1) == NULL) sCwd[0] = '\0'; + snprintf(sBuf, sizeof(sBuf), D2URI_LOCAL "%s/%s", sCwd, sFile); + return sBuf; +} + static int g_nFail = 0; static int g_nPass = 0; @@ -105,6 +117,15 @@ static void test_isUrlOrFile(void) CHECK(!das_isUrlOrFile("site:/uiowa"), "shorthand URI"); CHECK(!das_isUrlOrFile(NULL), "NULL"); CHECK(!das_isUrlOrFile(""), "empty"); + + printf("TEST: das_isUri\n"); + CHECK(das_isUri("tag:das2.org,2012:site:/uiowa"), "tag URI"); + CHECK(das_isUri("doi:10.1037/a0040251/"), "doi URI"); + CHECK(das_isUri("site:/uiowa") && das_isUri("test:x") && das_isUri("local:/x"), "shorthands"); + CHECK(das_isUri("https://host/x.json"), "a URL is a URI too"); + CHECK(!das_isUri("uiowa/juno/wav:x"), "relative path, colon after a slash"); + CHECK(!das_isUri("C:\\nodes\\x") && !das_isUri("C:/x"), "drive letters are not schemes"); + CHECK(!das_isUri("1tag:x") && !das_isUri(":x") && !das_isUri("") && !das_isUri(NULL), "not schemes"); } /* ************************************************************************* */ @@ -124,6 +145,10 @@ static void test_policy(void) CHECK(DasCat_setPrefix(pCat, "site:/uiowa") == DAS_OKAY, "prefix shorthand"); CHECK(streq(DasCat_prefix(pCat), UIOWA_URI), "prefix expanded: %s", DasCat_prefix(pCat)); + CHECK(DasCat_setPrefix(pCat, "site:uiowa") == DAS_OKAY, "prefix shorthand, slash left out"); + CHECK(streq(DasCat_prefix(pCat), UIOWA_URI), "slash supplied: %s", DasCat_prefix(pCat)); + CHECK(DasCat_setPrefix(pCat, "test:") == DAS_OKAY, "bare shorthand"); + CHECK(streq(DasCat_prefix(pCat), "tag:das2.org,2012:test:/"), "bare shorthand ends in a slash"); CHECK(DasCat_setPrefix(pCat, "tag:das2.org,2012:test:/") == DAS_OKAY, "prefix full"); CHECK(quiet(DasCat_setPrefix, pCat, SITE) != DAS_OKAY, "a file is not a prefix"); CHECK(quiet(DasCat_setPrefix, pCat, "") != DAS_OKAY, "empty prefix"); @@ -134,8 +159,12 @@ static void test_policy(void) CHECK(streq(DasCat_rootUrl(pCat), SITE), "root url kept"); CHECK(quiet2(DasCat_setRoot, pCat, UIOWA, NULL) != DAS_OKAY, "one root only"); CHECK(quiet2(DasCat_setRoot, pCat, UIOWA, UIOWA_URI) != DAS_OKAY, "no named root over a root"); - CHECK(quiet2(DasCat_mount, pCat, UIOWA, UIOWA_URI) != DAS_OKAY, "no mount over a root"); - CHECK(DasCat_rootUri(pCat) == NULL, "nameless root has no URI"); + CHECK(streq(DasCat_rootUri(pCat), localUri(SITE)), "a root without a URI is named by location: %s", + DasCat_rootUri(pCat)); + CHECK(DasCat_mount(pCat, UIOWA, UIOWA_URI) == DAS_OKAY, "a mount joins an offline catalog"); + CHECK(DasCat_mountCount(pCat) == 2, "the root is a mount too"); + CHECK(DasCat_covers(pCat, UIOWA_URI "/x") && !DasCat_covers(pCat, "tag:das2.org,2012:site:/swri"), + "covered by the mount, walled off elsewhere"); del_DasCat(pCat); pCat = new_DasCat(NULL, NULL); @@ -146,17 +175,20 @@ static void test_policy(void) CHECK(DasCat_setRoot(pCat, UIOWA, "site:/uiowa") == DAS_OKAY, "named root"); CHECK(streq(DasCat_rootUri(pCat), UIOWA_URI), "root URI expanded"); CHECK(DasCat_hasRoot(pCat), "a named root is a root"); - CHECK(quiet2(DasCat_mount, pCat, UIOWA, UIOWA_URI) != DAS_OKAY, "no mount over a named root"); CHECK(quiet2(DasCat_setRoot, pCat, SITE, NULL) != DAS_OKAY, "no second root"); del_DasCat(pCat); pCat = new_DasCat(NULL, NULL); CHECK(DasCat_mount(pCat, UIOWA, UIOWA_URI) == DAS_OKAY, "mount one"); - CHECK(DasCat_mount(pCat, SITE, "site:/swri") == DAS_OKAY, "mount two"); + CHECK(DasCat_mount(pCat, SITE, "site:swri") == DAS_OKAY, "mount two, slash left out"); CHECK(DasCat_mountCount(pCat) == 2, "two mounts"); CHECK(!DasCat_hasRoot(pCat), "mounts are not a root"); - CHECK(quiet2(DasCat_setRoot, pCat, UIOWA, UIOWA_URI) != DAS_OKAY, "no named root over mounts"); - CHECK(quiet2(DasCat_setRoot, pCat, SITE, NULL) != DAS_OKAY, "no root over mounts"); + CHECK(DasCat_covers(pCat, "tag:das2.org,2012:site:/swri/x"), "slash supplied in the mount URI"); + CHECK(DasCat_covers(pCat, "tag:das2.org,2012:site:/lanl"), "mounts fall through"); + CHECK(!DasCat_covers(pCat, "tag:das2.org,2012:local:/nowhere"), "local: never falls through"); + CHECK(DasCat_setRoot(pCat, SITE, "test:/") == DAS_OKAY, "a root after mounts isolates them"); + CHECK(DasCat_hasRoot(pCat) && (DasCat_mountCount(pCat) == 3), "three mounts, one is the root"); + CHECK(!DasCat_covers(pCat, "tag:das2.org,2012:site:/lanl"), "no fall through now"); del_DasCat(pCat); } @@ -167,11 +199,14 @@ static void test_rootMode(void) DasCat* pCat = new_DasCat(NULL, "TestCat"); DasCat_setCache(pCat, NULL, 0); /* explicitly off, file nodes never cache anyway */ CHECK(DasCat_setRoot(pCat, SITE, NULL) == DAS_OKAY, "set root"); + char sRootUri[1024]; + strcpy(sRootUri, localUri(SITE)); DasNode* pRoot = DasCat_resolve(pCat, NULL); CHECK(pRoot != NULL, "root resolves"); CHECK(pRoot && DasNode_isCatalog(pRoot), "root is a catalog"); - CHECK(streq(DasCat_lastUri(pCat), SITE), "no path: last URI is the root location"); + CHECK(streq(DasCat_lastUri(pCat), sRootUri), "no path: last URI is the root's local: name"); + CHECK(pRoot && streq(DasNode_pathUri(pRoot), sRootUri), "the node carries it"); CHECK(DasCat_resolve(pCat, "") == pRoot, "empty path is the same as NULL"); /* Policy is frozen now */ @@ -180,8 +215,13 @@ static void test_rootMode(void) DasNode* pSrc = DasCat_resolve(pCat, "uiowa/geomag/flex"); CHECK(pSrc != NULL, "deep path resolves"); CHECK(pSrc && DasNode_isStreamSrc(pSrc), "leaf is a stream source"); - CHECK(streq(DasCat_lastUri(pCat), "uiowa/geomag/flex"), "relative path is the URI under a URL root"); - CHECK(pSrc && streq(DasNode_pathUri(pSrc), "/uiowa/geomag/flex"), "node path URI: %s", + char sExpect[1100]; + /* site.json declares the ":/" separator of the node it stands in for, + and that is what joins its children to whatever name the root got */ + snprintf(sExpect, sizeof(sExpect), "%s:/uiowa/geomag/flex", sRootUri); + CHECK(streq(DasCat_lastUri(pCat), sExpect), "relative path hangs off the local: name: %s", + DasCat_lastUri(pCat)); + CHECK(pSrc && streq(DasNode_pathUri(pSrc), sExpect), "node path URI: %s", pSrc ? DasNode_pathUri(pSrc) : "(null)"); CHECK(DasCat_resolve(pCat, "uiowa/geomag/flex") == pSrc, "second resolve is the cached node"); @@ -192,13 +232,16 @@ static void test_rootMode(void) /* Misses, and what the catalog can say about them */ CHECK(quietResolve(pCat, "uiowa/nope/deeper") == NULL, "bad child fails"); - CHECK(streq(DasCat_lastUri(pCat), "uiowa/nope/deeper"), "last URI is the failed path"); - CHECK(DasCat_covers(pCat, DasCat_lastUri(pCat)), "a URL root covers everything"); + snprintf(sExpect, sizeof(sExpect), "%s:/uiowa/nope/deeper", sRootUri); + CHECK(streq(DasCat_lastUri(pCat), sExpect), "last URI is the failed path"); + CHECK(DasCat_covers(pCat, DasCat_lastUri(pCat)), "under the root is covered"); + CHECK(!DasCat_covers(pCat, UIOWA_URI), "the federated catalog is not"); const char* sRest = NULL; DasNode* pNear = DasCat_nearest(pCat, &sRest); CHECK(pNear == pUiowa, "nearest is the deepest catalog that loaded"); CHECK(streq(sRest, "nope/deeper"), "rest below it: %s", sRest ? sRest : "(null)"); - CHECK(streq(DasCat_lastUri(pCat), "uiowa"), "last URI now names the nearest node"); + snprintf(sExpect, sizeof(sExpect), "%s:/uiowa", sRootUri); + CHECK(streq(DasCat_lastUri(pCat), sExpect), "last URI now names the nearest node"); CHECK(quietResolve(pCat, "uiowa/broken") == NULL, "listed child with a missing file fails"); pNear = DasCat_nearest(pCat, &sRest); @@ -209,20 +252,34 @@ static void test_rootMode(void) CHECK(quietResolve(pCat, "nope") == NULL, "bad child of the root"); pNear = DasCat_nearest(pCat, &sRest); CHECK(pNear == NULL, "nothing above a root child"); - CHECK(streq(DasCat_lastUri(pCat), "nope"), "a failed nearest keeps the failed URI"); + snprintf(sExpect, sizeof(sExpect), "%s:/nope", sRootUri); + CHECK(streq(DasCat_lastUri(pCat), sExpect), "a failed nearest keeps the failed URI"); /* A good resolve leaves nothing for nearest to find */ CHECK(DasCat_resolve(pCat, "uiowa/geomag") != NULL, "source set resolves"); CHECK(DasCat_nearest(pCat, &sRest) == pUiowa, "nearest after success is the parent, by design"); - /* A location given as the path, named by itself */ + /* A location given as the path gets a local: name */ DasNode* pDirect = DasCat_resolve(pCat, UIOWA); CHECK(pDirect && DasNode_isCatalog(pDirect), "node by file path"); - CHECK(streq(DasCat_lastUri(pCat), UIOWA), "location is its own URI"); - CHECK(pDirect && streq(DasNode_pathUri(pDirect), UIOWA), "node path URI is the location"); + CHECK(streq(DasCat_lastUri(pCat), localUri(UIOWA)), "named by location: %s", DasCat_lastUri(pCat)); + CHECK(pDirect && streq(DasNode_pathUri(pDirect), localUri(UIOWA)), "node path URI is the local: name"); CHECK(DasCat_nearest(pCat, &sRest) == NULL, "nearest means nothing for a location"); + CHECK(DasCat_resolve(pCat, "./" UIOWA) != pDirect, "a different spelling is a different name"); + CHECK(streq(DasCat_lastUri(pCat), localUri(UIOWA)), "but a leading ./ is dropped: %s", DasCat_lastUri(pCat)); + + /* local: names that nothing is mounted at fail without a lookup */ + CHECK(quietResolve(pCat, "local:/nowhere/x") == NULL, "unmounted local: name fails"); + CHECK(streq(DasCat_lastUri(pCat), "tag:das2.org,2012:local:/nowhere/x"), "and was expanded first"); del_DasCat(pCat); + + /* No root at all: a local: name still never reaches the network */ + pCat = new_DasCat(NULL, "TestCat"); + CHECK(quietResolve(pCat, "tag:das2.org,2012:local:/home/nobody/x.json") == NULL, "unrouted"); + CHECK(!DasCat_covers(pCat, "local:/x") || true, "covers takes shorthand too"); + CHECK(!DasCat_covers(pCat, "tag:das2.org,2012:local:/x"), "and says it is not reachable"); + del_DasCat(pCat); } /* ************************************************************************* */ @@ -257,7 +314,20 @@ static void test_namedRoot(void) CHECK(pNear && DasNode_isCatalog(pNear) && streq(sRest, "nope"), "nearest under a named root: %s", sRest ? sRest : "(null)"); CHECK(streq(DasCat_lastUri(pCat), UIOWA_URI "/geomag"), "nearest URI under a named root"); + del_DasCat(pCat); + /* The site file as the site node: the bare shorthand names the node, + the node's own separator builds the children, so names come out as + the live catalog spells them */ + pCat = new_DasCat(NULL, "TestCat"); + CHECK(DasCat_setRoot(pCat, SITE, "site:") == DAS_OKAY, "root named by bare shorthand"); + CHECK(streq(DasCat_rootUri(pCat), D2URI_SITES), "bare shorthand is the node name: %s", DasCat_rootUri(pCat)); + pSrc = DasCat_resolve(pCat, "uiowa/geomag/flex"); + CHECK(pSrc && streq(DasNode_pathUri(pSrc), UIOWA_URI "/geomag/flex"), "children spelled as the live catalog: %s", + pSrc ? DasNode_pathUri(pSrc) : "(null)"); + CHECK(DasCat_resolve(pCat, "site:/uiowa/geomag/flex") == pSrc, "full URI reaches the same node"); + CHECK(DasCat_resolve(pCat, "site:uiowa/geomag/flex") == pSrc, "slash left out, same node"); + CHECK(DasCat_resolve(pCat, "site:") == DasCat_resolve(pCat, NULL), "bare shorthand resolves the root"); del_DasCat(pCat); } diff --git a/test/TestSettable.c b/test/TestSettable.c deleted file mode 100644 index e14de81..0000000 --- a/test/TestSettable.c +++ /dev/null @@ -1,415 +0,0 @@ -/* Copyright (C) 2026 Chris Piker - * - * This file is part of das2C, the Core Das2 C Library. - * - * Das2C is free software; you can redistribute it and/or modify it under - * the terms of the GNU Lesser General Public License version 2.1 as published - * by the Free Software Foundation. - * - * Das2C is distributed in the hope that it will be useful, but WITHOUT ANY - * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS - * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for - * more details. - * - * You should have received a copy of the GNU Lesser General Public License - * version 2.1 along with das2C; if not, see . - */ - -/* Source settings parsed straight from the three saved source nodes under - * test/das_fetch, no DasNode involved. Run from the repository top. - * - * The private parser is driven directly because that is the seam the - * source node load (step 6) will call; the public accessors are then - * checked against what the fixtures say. geomag is the current layout - * with examples, hsk is the current layout at scale (73 variables), and - * juno_survey is the 2018 layout with long property names and flag packed - * options. */ - -#define _POSIX_C_SOURCE 200112L - -#include -#include -#include -#include - -#include -#include -#include "../das3/sett_priv.h" - -#define GEOMAG "test/das_fetch/geomag_flex_0.7.json" -#define HSK "test/das_fetch/hsk_flex_0.7.json" -#define JUNO "test/das_fetch/juno_survey_0.6.json" - -static int g_nFail = 0; -static int g_nPass = 0; - -#define CHECK(cond, ...) do{ \ - if(cond){ ++g_nPass; } \ - else{ ++g_nFail; printf(" FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } \ -}while(0) - -static bool streq(const char* a, const char* b) -{ - if((a == NULL)||(b == NULL)) return (a == b); - return strcmp(a, b) == 0; -} - -/* Validation failures log at error level by design; keep the test output - readable while provoking them */ -static DasErrCode quietCheck(const DasSettable* p, const char* sVal) -{ - daslog_setlevel(DASLOG_CRIT); - DasErrCode n = DasSettable_check(p, sVal); - daslog_setlevel(DASLOG_WARN); - return n; -} - -static DasJdo* loadJson(const char* sFile) -{ - FILE* pIn = fopen(sFile, "rb"); - if(pIn == NULL){ printf(" Can't open %s, run from the repo top\n", sFile); return NULL; } - fseek(pIn, 0, SEEK_END); - long nLen = ftell(pIn); - fseek(pIn, 0, SEEK_SET); - char* sBuf = (char*)malloc(nLen + 1); - size_t uRead = fread(sBuf, 1, nLen, pIn); - fclose(pIn); - sBuf[uRead] = '\0'; - DasJdo* pDom = das_json_parse(sBuf, uRead); - free(sBuf); - return pDom; -} - -/* The settable list for a saved node, through the same two dicts the node - load will pass */ -static DasSettable* parseNode(const DasJdo* pDom, size_t* puCount) -{ - const DasJdo* pParams = DasJdo_get(pDom, "protocol/httpParams"); - if(pParams == NULL) pParams = DasJdo_get(pDom, "protocol/http_params"); - return _DasSettable_parse(DasJdo_get(pDom, "interface"), pParams, puCount); -} - -static const DasSettable* byKey(const DasSettable* aList, size_t uCount, const char* sKey) -{ - for(size_t u = 0; u < uCount; ++u) - if(streq(DasSettable_key(aList + u), sKey)) return aList + u; - return NULL; -} - -/* ************************************************************************* */ - -static void test_grammar(void) -{ - printf("Key grammar:\n"); - das_src_sec_e sec; - CHECK(_das_src_secByName("coord", &sec) && (sec == das_src_sec_coord), "coord"); - CHECK(_das_src_secByName("Coordinates", &sec) && (sec == das_src_sec_coord), "Coordinates"); - CHECK(_das_src_secByName("fmt", &sec) && (sec == das_src_sec_format), "fmt"); - CHECK(_das_src_secByName("opts", &sec) && (sec == das_src_sec_option), "opts"); - CHECK(!_das_src_secByName("time", &sec), "time is not a section"); - CHECK(!_das_src_secByName("", &sec), "empty is not a section"); - - CHECK(streq(das_src_sec_name(das_src_sec_coord), "coords"), "coords name"); - CHECK(streq(das_src_sec_name(das_src_sec_format), "formats"), "formats name"); - CHECK(streq(das_src_sec_name(das_src_sec_size), ""), "sentinel has no name"); - - CHECK(streq(_das_src_propName("Minimum"), "min"), "Minimum -> min"); - CHECK(streq(_das_src_propName("inter"), "inter"), "inter stays"); - CHECK(streq(_das_src_propName("enable"), "enabled"), "enable -> enabled"); - CHECK(streq(_das_src_propName("delim"), "delim"), "unknown passes through"); - CHECK(_das_src_propEq("maximum", "MAX"), "maximum == MAX"); - CHECK(!_das_src_propEq("res", "inter"), "res != inter"); - - char aParts[3][DAS_MAX_ID_BUFSZ]; - CHECK(_das_src_splitKey("coords.time.min", aParts) == 3, "three parts"); - CHECK(streq(aParts[1], "time") && streq(aParts[2], "min"), "parts content"); - CHECK(_das_src_splitKey("lshell", aParts) == 1, "one part"); - CHECK(_das_src_splitKey("a.b.c.d", aParts) == 0, "four parts is malformed"); - CHECK(_das_src_splitKey("a..c", aParts) == 0, "empty part is malformed"); - CHECK(_das_src_splitKey("", aParts) == 0, "empty key is malformed"); - CHECK(_das_src_splitKey(NULL, aParts) == 0, "NULL key is malformed"); -} - -static void test_scalar(void) -{ - printf("JSON scalars:\n"); - /* Two letter keys: DasJdo_get() does not resolve one character names */ - const char* sDoc = "{\"st\":\"x\",\"nm\":86.4,\"in\":42,\"tr\":true,\"fa\":false," - "\"nl\":null,\"ar\":[1],\"di\":{}}"; - DasJdo* pDom = das_json_parse(sDoc, strlen(sDoc)); - CHECK(pDom != NULL, "parse"); - char sBuf[32]; - CHECK(_das_json_scalar(DasJdo_get(pDom, "st"), sBuf, 32) && streq(sBuf, "x"), "string"); - CHECK(_das_json_scalar(DasJdo_get(pDom, "nm"), sBuf, 32) && streq(sBuf, "86.4"), "number text kept"); - CHECK(_das_json_scalar(DasJdo_get(pDom, "in"), sBuf, 32) && streq(sBuf, "42"), "integer"); - CHECK(_das_json_scalar(DasJdo_get(pDom, "tr"), sBuf, 32) && streq(sBuf, "true"), "true"); - CHECK(_das_json_scalar(DasJdo_get(pDom, "fa"), sBuf, 32) && streq(sBuf, "false"), "false"); - CHECK(!_das_json_scalar(DasJdo_get(pDom, "nl"), sBuf, 32), "null is not a scalar"); - CHECK(!_das_json_scalar(DasJdo_get(pDom, "ar"), sBuf, 32), "array is not a scalar"); - CHECK(!_das_json_scalar(DasJdo_get(pDom, "di"), sBuf, 32), "dict is not a scalar"); - CHECK(!_das_json_scalar(NULL, sBuf, 32), "NULL is not a scalar"); - CHECK(_das_json_scalar(DasJdo_get(pDom, "nm"), sBuf, 3) && streq(sBuf, "86"), "number cut to fit"); - free(pDom); -} - -static void test_geomag(void) -{ - printf("geomag, current layout:\n"); - DasJdo* pDom = loadJson(GEOMAG); - CHECK(pDom != NULL, "load"); - if(pDom == NULL) return; - - size_t uCount = 0; - DasSettable* aList = parseNode(pDom, &uCount); - CHECK(aList != NULL, "parse"); - - /* time min/max/inter, lshell enabled, then 13 format knobs */ - CHECK(uCount == 17, "17 settables, got %zu", uCount); - - /* Order is section then document order */ - CHECK((uCount > 4) && (DasSettable_section(aList) == das_src_sec_coord), "coords first"); - CHECK((uCount > 4) && (DasSettable_section(aList + 3) == das_src_sec_data), "data second"); - CHECK((uCount > 4) && (DasSettable_section(aList + 4) == das_src_sec_format), "formats third"); - - const DasSettable* p = byKey(aList, uCount, "coords.time.min"); - CHECK(p != NULL, "coords.time.min"); - if(p){ - CHECK(streq(DasSettable_group(p), "time"), "group"); - CHECK(streq(DasSettable_prop(p), "min"), "prop"); - CHECK(streq(DasSettable_param(p), "read.time.min"), "param via dotted name"); - CHECK(DasSettable_required(p), "required from the protocol parameter"); - CHECK(DasSettable_type(p) == das_settable_time, "type from the parameter"); - CHECK(streq(DasSettable_typeName(p), "isotime"), "type name from the parameter"); - CHECK(streq(DasSettable_units(p), "UTC"), "UTC for a time"); - CHECK(DasSettable_defVal(p) == NULL, "null value is no default"); - CHECK(quietCheck(p, "2026-01-01") == DAS_OKAY, "ISO date passes"); - CHECK(quietCheck(p, "2026-01-01T12:00:00") == DAS_OKAY, "ISO time passes"); - CHECK(quietCheck(p, "now") == DAS_OKAY, "now passes"); - CHECK(quietCheck(p, "yesterday-ish") == DASERR_SRC, "garbage fails"); - } - - p = byKey(aList, uCount, "coords.time.inter"); - CHECK(p != NULL, "coords.time.inter"); - if(p){ - CHECK(DasSettable_type(p) == das_settable_real, "real from the parameter"); - CHECK(streq(DasSettable_units(p), "s"), "units string on the property"); - CHECK(quietCheck(p, "60") == DAS_OKAY, "60 is real"); - CHECK(quietCheck(p, "6e1") == DAS_OKAY, "6e1 is real"); - CHECK(quietCheck(p, "sixty") == DASERR_SRC, "sixty is not"); - } - - p = byKey(aList, uCount, "data.lshell.enabled"); - CHECK(p != NULL, "data.lshell.enabled"); - if(p){ - CHECK(DasSettable_type(p) == das_settable_bool, "boolean"); - CHECK(streq(DasSettable_param(p), "read.data"), "flag set parameter"); - CHECK(!DasSettable_required(p), "not required"); - CHECK(quietCheck(p, "yes") == DAS_OKAY, "yes"); - CHECK(quietCheck(p, "OFF") == DAS_OKAY, "OFF"); - CHECK(quietCheck(p, "maybe") == DASERR_SRC, "maybe"); - } - - char sBuf[64]; - p = byKey(aList, uCount, "formats.csv.enabled"); - CHECK(p != NULL, "formats.csv.enabled"); - if(p){ - CHECK(streq(DasSettable_param(p), "format.type"), "format selector param"); - const char* sTitle = DasSettable_title(p, sBuf, sizeof(sBuf)); - CHECK(strncmp(sTitle, "Output as ", 10) == 0, "composed format title, got '%s'", sTitle); - } - - p = byKey(aList, uCount, "formats.csv.delimiter"); - CHECK(p != NULL, "formats.csv.delimiter"); - if(p){ - CHECK(DasSettable_type(p) == das_settable_enum, "delimiter is an enum, got %d", DasSettable_type(p)); - CHECK(DasSettable_enumCount(p) == 4, "four delimiters, got %zu", DasSettable_enumCount(p)); - CHECK(DasSettable_enum(p, 4) == NULL, "past the end"); - CHECK(streq(DasSettable_enum(p, 0), "Comma"), "node spelling kept"); - CHECK(quietCheck(p, DasSettable_enum(p, 0)) == DAS_OKAY, "first choice passes"); - CHECK(quietCheck(p, "tab") == DAS_OKAY, "enum match is case insensitive"); - CHECK(quietCheck(p, "colon") == DASERR_SRC, "colon is not offered"); - } - - p = byKey(aList, uCount, "formats.csv.fracSecs"); - CHECK(p != NULL, "formats.csv.fracSecs"); - if(p){ - const char* sLo = NULL; const char* sHi = NULL; - CHECK(DasSettable_type(p) == das_settable_int, "integer from the parameter"); - CHECK(streq(DasSettable_param(p), "format.secfrac"), "param"); - CHECK(DasSettable_range(p, &sLo, &sHi), "range from the parameter"); - CHECK(streq(sLo, "0") && streq(sHi, "9"), "0 to 9, got %s to %s", sLo, sHi); - CHECK(quietCheck(p, "3") == DAS_OKAY, "3"); - CHECK(quietCheck(p, "12") == DAS_OKAY, "12 passes, ranges are not enforced"); - CHECK(quietCheck(p, "3.5") == DASERR_SRC, "3.5 is not an integer"); - } - - _DasSettable_freeList(aList, uCount); - free(pDom); -} - -static void test_hsk(void) -{ - printf("hsk, current layout at scale:\n"); - DasJdo* pDom = loadJson(HSK); - CHECK(pDom != NULL, "load"); - if(pDom == NULL) return; - - size_t uCount = 0; - DasSettable* aList = parseNode(pDom, &uCount); - CHECK(aList != NULL, "parse"); - CHECK(uCount == 90, "3 + 73 + 14 settables, got %zu", uCount); - - size_t uData = 0; - for(size_t u = 0; u < uCount; ++u) - if(DasSettable_section(aList + u) == das_src_sec_data) ++uData; - CHECK(uData == 73, "73 variable toggles, got %zu", uData); - - /* Every entry has a key, a parameter, and a printable title */ - bool bAllKeyed = true, bAllParam = true, bAllTitled = true; - char sBuf[64]; - for(size_t u = 0; u < uCount; ++u){ - if(DasSettable_key(aList + u)[0] == '\0') bAllKeyed = false; - if(DasSettable_param(aList + u) == NULL) bAllParam = false; - if(DasSettable_title(aList + u, sBuf, sizeof(sBuf)) == NULL) bAllTitled = false; - } - CHECK(bAllKeyed, "all keyed"); - CHECK(bAllParam, "all have a parameter"); - CHECK(bAllTitled, "all titled"); - - const DasSettable* p = byKey(aList, uCount, "coords.time.min"); - CHECK(p != NULL, "coords.time.min"); - if(p){ - const char* sLo = NULL; const char* sHi = NULL; - CHECK(DasSettable_range(p, &sLo, &sHi), "coordinate validRange reaches min"); - CHECK(sLo && (strncmp(sLo, "2025-", 5) == 0), "range low is the mission start, got %s", sLo); - CHECK(streq(DasSettable_title(p, sBuf, sizeof(sBuf)), "Minimum time value to stream"), - "node title wins over the composed one"); - CHECK(streq(DasSettable_units(p), "UTC"), "UTC from the group units"); - } - p = byKey(aList, uCount, "coords.time.res"); - CHECK(p != NULL, "coords.time.res"); - if(p){ - CHECK(!DasSettable_range(p, NULL, NULL), "res has no range"); - CHECK(streq(DasSettable_units(p), "s"), "seconds"); - CHECK(!DasSettable_required(p), "optional"); - } - p = byKey(aList, uCount, "formats.csv.headers"); - CHECK(p != NULL, "formats.csv.headers"); - if(p){ - CHECK(streq(DasSettable_defVal(p), "true"), "default true"); - CHECK(DasSettable_type(p) == das_settable_bool, "boolean"); - } - - _DasSettable_freeList(aList, uCount); - free(pDom); -} - -static void test_juno(void) -{ - printf("juno_survey, 2018 layout:\n"); - DasJdo* pDom = loadJson(JUNO); - CHECK(pDom != NULL, "load"); - if(pDom == NULL) return; - - size_t uCount = 0; - DasSettable* aList = parseNode(pDom, &uCount); - CHECK(aList != NULL, "parse"); - - /* time minimum/maximum/resolution and the options block */ - size_t uCoord = 0, uOpt = 0; - for(size_t u = 0; u < uCount; ++u){ - if(DasSettable_section(aList + u) == das_src_sec_coord) ++uCoord; - if(DasSettable_section(aList + u) == das_src_sec_option) ++uOpt; - } - CHECK(uCoord == 3, "three time settables, got %zu", uCoord); - CHECK(uOpt >= 8, "flag options found, got %zu", uOpt); - CHECK(uCount == uCoord + uOpt, "nothing settable in data, got %zu", uCount); - - /* Keys use the node's long spelling, matching folds it */ - const DasSettable* p = byKey(aList, uCount, "coords.time.minimum"); - CHECK(p != NULL, "coords.time.minimum keyed as spelled"); - CHECK(byKey(aList, uCount, "coords.time.min") == NULL, "not rewritten to min"); - if(p){ - CHECK(_das_src_propEq(DasSettable_prop(p), "min"), "propEq folds it"); - CHECK(streq(DasSettable_param(p), "start_time"), "param"); - CHECK(DasSettable_required(p), "required from the set block"); - CHECK(streq(DasSettable_defVal(p), "2018-10-02T11:00"), "default kept"); - const char* sLo = NULL; const char* sHi = NULL; - CHECK(DasSettable_range(p, &sLo, &sHi) && streq(sHi, "now"), "set range to now"); - CHECK(DasSettable_type(p) == das_settable_time, "type from the old parameter dict"); - CHECK(streq(DasSettable_typeName(p), "isotime"), "type name from the parameter"); - CHECK(quietCheck(p, "2018-10-02T11:00") == DAS_OKAY, "the default passes its own check"); - CHECK(streq(DasSettable_units(p), "UTC"), "UTC from the old units dict"); - char sBuf[64]; - CHECK(streq(DasSettable_title(p, sBuf, sizeof(sBuf)), "Minimum Time value to output"), - "composed from the group name, got '%s'", DasSettable_title(p, sBuf, sizeof(sBuf))); - } - p = byKey(aList, uCount, "coords.time.resolution"); - CHECK(p != NULL, "coords.time.resolution"); - if(p){ - CHECK(streq(DasSettable_defVal(p), "86.4"), "number default in source notation"); - CHECK(streq(DasSettable_units(p), "s"), "units string on the property"); - } - - /* Options: two part keys, empty group, flag toggles are booleans */ - p = byKey(aList, uCount, "options.lfrl"); - CHECK(p != NULL, "options.lfrl"); - if(p){ - CHECK(streq(DasSettable_group(p), ""), "options have no group"); - CHECK(DasSettable_type(p) == das_settable_bool, "flag toggle is boolean"); - CHECK(streq(DasSettable_param(p), "params"), "packed parameter"); - CHECK(streq(DasSettable_defVal(p), "false"), "default false"); - char sBuf[64]; - CHECK(strlen(DasSettable_title(p, sBuf, sizeof(sBuf))) > 10, "node title"); - } - p = byKey(aList, uCount, "options.das2_text"); - CHECK(p != NULL, "options.das2_text"); - if(p) CHECK(DasSettable_type(p) == das_settable_bool, "boolean from a boolean default"); - - _DasSettable_freeList(aList, uCount); - free(pDom); -} - -static void test_edges(void) -{ - printf("Edge cases:\n"); - size_t uCount = 99; - DasSettable* aList = _DasSettable_parse(NULL, NULL, &uCount); - CHECK((aList != NULL) && (uCount == 0), "NULL interface is an empty list"); - _DasSettable_freeList(aList, uCount); - - /* A set block with no param is dropped with a warning, the rest stays */ - const char* sDoc = - "{\"interface\":{\"options\":{" - " \"good\":{\"value\":false,\"set\":{\"param\":\"g\",\"value\":true}}," - " \"broken\":{\"value\":false,\"set\":{\"value\":true}}," - " \"doc_only\":{\"value\":\"x\"}" - "}}}"; - DasJdo* pDom = das_json_parse(sDoc, strlen(sDoc)); - CHECK(pDom != NULL, "parse edge doc"); - daslog_setlevel(DASLOG_ERROR); - aList = _DasSettable_parse(DasJdo_get(pDom, "interface"), NULL, &uCount); - daslog_setlevel(DASLOG_WARN); - CHECK(uCount == 1, "one settable survives, got %zu", uCount); - CHECK((uCount == 1) && streq(DasSettable_key(aList), "options.good"), "the good one"); - CHECK((uCount == 1) && (DasSettable_type(aList) == das_settable_bool), "boolean by default value"); - CHECK((uCount == 1) && streq(DasSettable_title(aList, NULL, 0), ""), "untitled option is empty"); - _DasSettable_freeList(aList, uCount); - free(pDom); - _DasSettable_freeList(NULL, 0); -} - -int main(int argc, char** argv) -{ - das_init(argv[0], DASERR_DIS_EXIT, 0, DASLOG_WARN, NULL); - - test_grammar(); - test_scalar(); - test_geomag(); - test_hsk(); - test_juno(); - test_edges(); - - printf("TestSettable: %d checks passed, %d failed\n", g_nPass, g_nFail); - return (g_nFail == 0) ? 0 : 1; -} diff --git a/test/TestSource.c b/test/TestSource.c new file mode 100644 index 0000000..2920988 --- /dev/null +++ b/test/TestSource.c @@ -0,0 +1,810 @@ +/* Copyright (C) 2026 Chris Piker + * + * This file is part of das2C, the Core Das2 C Library. + * + * Das2C is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License version 2.1 as published + * by the Free Software Foundation. + * + * Das2C is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for + * more details. + * + * You should have received a copy of the GNU Lesser General Public License + * version 2.1 along with das2C; if not, see . + */ + +/* Sources, requisitions and URLs from the three saved source nodes under + * test/das_fetch, entirely offline. Run from the repository top. + * + * geomag is the current layout with examples and mirrors, hsk the current + * layout at scale (73 variables), juno_survey the 2018 layout with long + * property names, flag packed options and protocol level examples. The + * URL cases are the ones utilities/das_fetch.c has always been held to, + * now asked of the library directly. */ + +#define _POSIX_C_SOURCE 200112L + +#include +#include +#include +#include + +#include +#include +#include +#include "../das3/src_priv.h" + +#define GEOMAG "test/das_fetch/geomag_flex_0.7.json" +#define HSK "test/das_fetch/hsk_flex_0.7.json" +#define JUNO "test/das_fetch/juno_survey_0.6.json" + +#define GB "http://127.0.0.1:8765/data/geomag.d2s" +#define HB "https://space.physics.uiowa.edu/das/stream/source/tracers/tracers-2/l1a/cdpu/hsk/flex" +#define JB "http://127.0.0.1:8765/data/juno_survey.d2s?server=dataset&dataset=Juno/WAV/Survey" + +static int g_nFail = 0; +static int g_nPass = 0; + +#define CHECK(cond, ...) do{ \ + if(cond){ ++g_nPass; } \ + else{ ++g_nFail; printf(" FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } \ +}while(0) + +static bool streq(const char* a, const char* b) +{ + if((a == NULL)||(b == NULL)) return (a == b); + return strcmp(a, b) == 0; +} + +static char* readFile(const char* sFile, size_t* puLen) +{ + FILE* pIn = fopen(sFile, "rb"); + if(pIn == NULL){ printf(" Can't open %s, run from the repo top\n", sFile); return NULL; } + fseek(pIn, 0, SEEK_END); + long nLen = ftell(pIn); + fseek(pIn, 0, SEEK_SET); + char* sBuf = (char*)malloc(nLen + 1); + *puLen = fread(sBuf, 1, nLen, pIn); + fclose(pIn); + sBuf[*puLen] = '\0'; + return sBuf; +} + +static DasSrc* loadSrc(const char* sFile, const char* sPath) +{ + size_t uLen = 0; + char* sDoc = readFile(sFile, &uLen); + if(sDoc == NULL) return NULL; + DasSrc* pSrc = new_DasSrc_json(sDoc, uLen, sPath, sFile); + free(sDoc); + return pSrc; +} + +static const DasSrcProp* byKey(const DasSrc* pSrc, const char* sKey) +{ + for(size_t u = 0; u < DasSrc_propCount(pSrc); ++u) + if(streq(DasSrcProp_key(DasSrc_prop(pSrc, u)), sKey)) return DasSrc_prop(pSrc, u); + return NULL; +} + +static const DasSrcCtrl* ctrlByKey(const DasSrc* pSrc, const char* sKey) +{ + const DasSrcProp* p = byKey(pSrc, sKey); + return p ? DasSrcProp_ctrl(p) : NULL; +} + +static size_t ctrlCount(const DasSrc* pSrc, int nSec) +{ + size_t n = 0; + for(size_t u = 0; u < DasSrc_propCount(pSrc); ++u){ + const DasSrcProp* p = DasSrc_prop(pSrc, u); + if((nSec >= 0) && (DasSrcProp_section(p) != (das_src_sec_e)nSec)) continue; + if(DasSrcProp_ctrl(p) != NULL) ++n; + } + return n; +} + +static DasErrCode chk(const DasSrcCtrl* pC, const char* sVal) +{ + return DasSrcCtrl_check(pC, sVal, NULL, 0); +} + +/* A KEY=VAL list to a URL, the das_fetch_test.sh shape. nExample seeds the + requisition, -1 for none. Errors are expected in some cases, so the log + is quiet for the duration. */ +static bool urlFor( + const DasSrc* pSrc, int nExample, const char** aKeyVal, const char** aRaw, + char* sUrl, size_t uLen +){ + DasReq* pReq = (nExample >= 0) ? DasSrc_newReq(pSrc, nExample) : new_DasReq(); + if(pReq == NULL) return false; + for(size_t u = 0; aKeyVal && aKeyVal[u]; ++u){ + const char* pEq = strchr(aKeyVal[u], '='); + char sKey[128]; + if(pEq){ + size_t n = (size_t)(pEq - aKeyVal[u]); + memcpy(sKey, aKeyVal[u], n); sKey[n] = '\0'; + DasReq_set(pReq, sKey, pEq + 1); + } + else{ + DasReq_set(pReq, aKeyVal[u], "true"); + } + } + for(size_t u = 0; aRaw && aRaw[u]; u += 2) DasReq_setRaw(pReq, aRaw[u], aRaw[u+1]); + daslog_setlevel(DASLOG_CRIT); + size_t n = DasStreamSrc_url(pSrc, pReq, 0, sUrl, uLen); + daslog_setlevel(DASLOG_WARN); + del_DasReq(pReq); + return n > 0; +} + +#define URL_IS(src, ex, keys, raw, want) do{ \ + char _s[2048]; const char* _k[] = keys; const char* _r[] = raw; \ + bool _ok = urlFor(src, ex, _k, _r, _s, sizeof(_s)); \ + CHECK(_ok && streq(_s, want), "\n want %s\n got %s", want, _ok ? _s : "(no url)"); \ +}while(0) + +#define URL_FAILS(src, ex, keys) do{ \ + char _s[2048]; const char* _k[] = keys; \ + CHECK(!urlFor(src, ex, _k, NULL, _s, sizeof(_s)), "expected failure, got %s", _s); \ +}while(0) + +#define K(...) {__VA_ARGS__, NULL} +#define NONE {NULL} + +/* ************************************************************************* */ + +static void test_grammar(void) +{ + printf("Key grammar:\n"); + das_src_sec_e sec; + CHECK(_das_src_secByName("coord", &sec) && (sec == das_src_sec_coord), "coord"); + CHECK(_das_src_secByName("Coordinates", &sec) && (sec == das_src_sec_coord), "Coordinates"); + CHECK(_das_src_secByName("fmt", &sec) && (sec == das_src_sec_format), "fmt"); + CHECK(_das_src_secByName("opts", &sec) && (sec == das_src_sec_option), "opts"); + CHECK(!_das_src_secByName("time", &sec), "time is not a section"); + CHECK(!_das_src_secByName("", &sec), "empty is not a section"); + + CHECK(streq(das_src_sec_name(das_src_sec_coord), "coords"), "coords name"); + CHECK(streq(das_src_sec_name(das_src_sec_format), "formats"), "formats name"); + CHECK(streq(das_src_sec_name(das_src_sec_size), ""), "sentinel has no name"); + + CHECK(streq(_das_src_propName("Minimum"), "min"), "Minimum -> min"); + CHECK(streq(_das_src_propName("inter"), "inter"), "inter stays"); + CHECK(streq(_das_src_propName("enable"), "enabled"), "enable -> enabled"); + CHECK(streq(_das_src_propName("delim"), "delim"), "unknown passes through"); + CHECK(_das_src_propEq("maximum", "MAX"), "maximum == MAX"); + CHECK(!_das_src_propEq("res", "inter"), "res != inter"); + + char aParts[3][DAS_MAX_ID_BUFSZ]; + CHECK(_das_src_splitKey("coords.time.min", aParts) == 3, "three parts"); + CHECK(streq(aParts[1], "time") && streq(aParts[2], "min"), "parts content"); + CHECK(_das_src_splitKey("lshell", aParts) == 1, "one part"); + CHECK(_das_src_splitKey("a.b.c.d", aParts) == 0, "four parts is malformed"); + CHECK(_das_src_splitKey("a..c", aParts) == 0, "empty part is malformed"); + CHECK(_das_src_splitKey("", aParts) == 0, "empty key is malformed"); + CHECK(_das_src_splitKey(NULL, aParts) == 0, "NULL key is malformed"); +} + +static void test_scalar(void) +{ + printf("JSON scalars:\n"); + /* Two letter keys: DasJdo_get() does not resolve one character names */ + const char* sDoc = "{\"st\":\"x\",\"nm\":86.4,\"in\":42,\"tr\":true,\"fa\":false," + "\"nl\":null,\"ar\":[1],\"di\":{}}"; + DasJdo* pDom = das_json_parse(sDoc, strlen(sDoc)); + CHECK(pDom != NULL, "parse"); + char sBuf[32]; + CHECK(_das_json_scalar(DasJdo_get(pDom, "st"), sBuf, 32) && streq(sBuf, "x"), "string"); + CHECK(_das_json_scalar(DasJdo_get(pDom, "nm"), sBuf, 32) && streq(sBuf, "86.4"), "number text kept"); + CHECK(_das_json_scalar(DasJdo_get(pDom, "in"), sBuf, 32) && streq(sBuf, "42"), "integer"); + CHECK(_das_json_scalar(DasJdo_get(pDom, "tr"), sBuf, 32) && streq(sBuf, "true"), "true"); + CHECK(_das_json_scalar(DasJdo_get(pDom, "fa"), sBuf, 32) && streq(sBuf, "false"), "false"); + CHECK(!_das_json_scalar(DasJdo_get(pDom, "nl"), sBuf, 32), "null is not a scalar"); + CHECK(!_das_json_scalar(DasJdo_get(pDom, "ar"), sBuf, 32), "array is not a scalar"); + CHECK(!_das_json_scalar(NULL, sBuf, 32), "NULL is not a scalar"); + CHECK(_das_json_scalar(DasJdo_get(pDom, "nm"), sBuf, 3) && streq(sBuf, "86"), "number cut to fit"); + free(pDom); +} + +static void test_helpers(void) +{ + printf("URL encoding and time spans:\n"); + char sBuf[64]; + CHECK(das_url_encode("2026-01-02T14:30", sBuf, sizeof(sBuf)) == 18, "encoded length"); + CHECK(streq(sBuf, "2026-01-02T14%3A30"), "colon encoded, got %s", sBuf); + CHECK(das_url_encode("a b&c", sBuf, sizeof(sBuf)) == 9, "space and ampersand"); + CHECK(streq(sBuf, "a%20b%26c"), "got %s", sBuf); + CHECK(das_url_encode("abc", sBuf, 2) == 3, "needed length when short"); + CHECK(streq(sBuf, "a"), "cut short and terminated, got '%s'", sBuf); + CHECK(das_url_encode("abc", NULL, 0) == 3, "measure only"); + CHECK(das_url_encode("", sBuf, sizeof(sBuf)) == 0 && streq(sBuf, ""), "empty"); + + das_time dtB, dtE; + char sB[40], sE[40]; +#define SPAN_IS(str, b, e) do{ \ + bool _ok = dt_parsespan(str, &dtB, &dtE); \ + if(_ok){ dt_isoc(&dtB, sB, 40, 0); dt_isoc(&dtE, sE, 40, 0); } \ + CHECK(_ok && streq(sB, b) && streq(sE, e), "%s -> %s %s, want %s %s", str, \ + _ok ? sB : "(refused)", _ok ? sE : "", b, e); \ +}while(0) + SPAN_IS("2026", "2026-01-01T00:00:00", "2027-01-01T00:00:00"); + SPAN_IS("2026-02", "2026-02-01T00:00:00", "2026-03-01T00:00:00"); + SPAN_IS("2026-01-02", "2026-01-02T00:00:00", "2026-01-03T00:00:00"); + SPAN_IS("2026-12-31", "2026-12-31T00:00:00", "2027-01-01T00:00:00"); + SPAN_IS("2026-01-02T14", "2026-01-02T14:00:00", "2026-01-02T15:00:00"); + SPAN_IS("2026-01-02T23", "2026-01-02T23:00:00", "2026-01-03T00:00:00"); + SPAN_IS("2026-01-02T14:30", "2026-01-02T14:30:00", "2026-01-02T14:31:00"); + SPAN_IS("2026-01-02T14:30:15Z", "2026-01-02T14:30:15", "2026-01-02T14:30:16"); + SPAN_IS("2026-002", "2026-01-02T00:00:00", "2026-01-03T00:00:00"); + SPAN_IS("2026-002T06", "2026-01-02T06:00:00", "2026-01-02T07:00:00"); + CHECK(!dt_parsespan("2026-01-02T14:30:15.5", NULL, NULL), "fractions have no span"); + CHECK(!dt_parsespan("T14", NULL, NULL), "a time needs a date"); + CHECK(!dt_parsespan("now", NULL, NULL), "now is not a span"); + CHECK(!dt_parsespan("", NULL, NULL), "empty"); + CHECK(!dt_parsespan("2026-01-02,2026-01-03", NULL, NULL), "a range is not a span"); + CHECK(dt_parsespan("2026-01-02", NULL, NULL), "outputs are optional"); +} + +static void test_geomag(void) +{ + printf("geomag, current layout:\n"); + DasSrc* pSrc = loadSrc(GEOMAG, "tag:das2.org,2012:site:/uiowa/tracers/tracers-2/spice/geomagcoords/flex"); + CHECK(pSrc != NULL, "load"); + if(pSrc == NULL) return; + + CHECK(DasSrc_kind(pSrc) == das_src_kind_stream, "stream source"); + CHECK(streq(DasSrc_version(pSrc), "0.7"), "version"); + CHECK(streq(DasSrc_url(pSrc), GEOMAG), "url is where it came from"); + CHECK(!DasSrc_needsAuth(pSrc), "no auth"); + CHECK(DasStreamSrc_urlCount(pSrc) == 2, "two mirrors"); + CHECK(streq(DasStreamSrc_baseUrl(pSrc, 1), "http://127.0.0.1:8765/data/geomag_alt.d2s"), "second mirror"); + CHECK(DasStreamSrc_baseUrl(pSrc, 2) == NULL, "past the mirrors"); + + /* time min/max/inter, lshell enabled, then 13 format knobs */ + CHECK(ctrlCount(pSrc, -1) == 17, "17 controls, got %zu", ctrlCount(pSrc, -1)); + CHECK(DasSrc_propCount(pSrc) > 17, "fixed props are kept too, %zu total", DasSrc_propCount(pSrc)); + + /* Groups */ + CHECK(DasSrc_groupCount(pSrc, das_src_sec_coord) == 1, "one coordinate"); + CHECK(DasSrc_groupCount(pSrc, das_src_sec_data) == 1, "one variable"); + CHECK(DasSrc_groupCount(pSrc, das_src_sec_format) == 3, "three formats, got %zu", + DasSrc_groupCount(pSrc, das_src_sec_format)); + CHECK(DasSrc_groupCount(pSrc, das_src_sec_option) == 0, "no options"); + CHECK(DasSrc_group(pSrc, das_src_sec_option, 0) == NULL, "no option group"); + + const DasSrcGroup* pG = DasSrc_findGroup(pSrc, das_src_sec_coord, "time"); + CHECK(pG != NULL, "time group"); + if(pG){ + CHECK(streq(DasSrcGroup_label(pG), "Time"), "label"); + CHECK(DasSrcGroup_enable(pG) == NULL, "coordinates have no gate"); + CHECK(DasSrcGroup_propCount(pG) == 4, "min max units inter, got %zu", DasSrcGroup_propCount(pG)); + CHECK(DasSrcGroup_ctrlCount(pG) == 3, "three of them settable"); + const DasSrcProp* pU = DasSrcGroup_findProp(pG, "units"); + CHECK(pU && (DasSrcProp_ctrl(pU) == NULL), "units is a fixed property"); + CHECK(pU && streq(DasSrcProp_value(pU), "UTC"), "with the value UTC"); + CHECK(DasSrcGroup_findProp(pG, "minimum") == DasSrcGroup_findProp(pG, "min"), "alias in findProp"); + CHECK(DasSrcGroup_prop(pG, 4) == NULL, "past the end"); + } + + pG = DasSrc_findGroup(pSrc, das_src_sec_format, "csv"); + CHECK(pG != NULL, "csv group"); + if(pG){ + const DasSrcProp* pOn = DasSrcGroup_enable(pG); + CHECK(pOn && streq(DasSrcProp_key(pOn), "formats.csv.enabled"), "the gate"); + CHECK(streq(DasSrcGroup_xorGroup(pG), "format"), "formats are exclusive"); + CHECK(DasSrcGroup_mimeCount(pG) == 1 && streq(DasSrcGroup_mime(pG, 0), "text/csv"), "mime"); + } + pG = DasSrc_findGroup(pSrc, das_src_sec_data, "lshell"); + CHECK(pG && DasSrcGroup_enable(pG) && (DasSrcGroup_xorGroup(pG) == NULL), "variable gate, not exclusive"); + + /* Properties and controls */ + const DasSrcProp* p = byKey(pSrc, "coords.time.min"); + CHECK(p != NULL, "coords.time.min"); + if(p){ + const DasSrcCtrl* pC = DasSrcProp_ctrl(p); + CHECK(pC != NULL, "has a control"); + CHECK(DasSrcCtrl_prop(pC) == p, "control points back"); + CHECK(streq(DasSrcGroup_name(DasSrcProp_group(p)), "time"), "group"); + CHECK(streq(DasSrcProp_name(p), "min"), "name"); + CHECK(streq(DasSrcCtrl_param(pC), "read.time.min"), "param via dotted name"); + CHECK(DasSrcCtrl_required(pC), "required from the protocol parameter"); + CHECK(DasSrcProp_type(p) == das_srcprop_time, "type from the parameter"); + CHECK(streq(DasSrcProp_typeName(p), "isotime"), "type name from the parameter"); + CHECK(streq(DasSrcProp_units(p), "UTC"), "UTC for a time"); + CHECK(DasSrcProp_value(p) == NULL, "null value is no default"); + CHECK(chk(pC, "2026-01-01") == DAS_OKAY, "ISO date passes"); + CHECK(chk(pC, "2026-01-01T12:00:00") == DAS_OKAY, "ISO time passes"); + CHECK(chk(pC, "now") == DAS_OKAY, "now passes"); + char sWhy[256]; + CHECK(DasSrcCtrl_check(pC, "yesterday-ish", sWhy, sizeof(sWhy)) == DASERR_SRC, "garbage fails"); + CHECK(strstr(sWhy, "coords.time.min") != NULL, "reason names the key: %s", sWhy); + } + + const DasSrcCtrl* pC = ctrlByKey(pSrc, "coords.time.inter"); + CHECK(pC != NULL, "coords.time.inter"); + if(pC){ + CHECK(DasSrcProp_type(DasSrcCtrl_prop(pC)) == das_srcprop_real, "real from the parameter"); + CHECK(streq(DasSrcProp_units(DasSrcCtrl_prop(pC)), "s"), "units string on the property"); + CHECK(chk(pC, "60") == DAS_OKAY, "60 is real"); + CHECK(chk(pC, "6e1") == DAS_OKAY, "6e1 is real"); + CHECK(chk(pC, "sixty") == DASERR_SRC, "sixty is not"); + } + + pC = ctrlByKey(pSrc, "data.lshell.enabled"); + CHECK(pC != NULL, "data.lshell.enabled"); + if(pC){ + CHECK(DasSrcProp_type(DasSrcCtrl_prop(pC)) == das_srcprop_bool, "boolean"); + CHECK(streq(DasSrcCtrl_param(pC), "read.data"), "flag set parameter"); + CHECK(!DasSrcCtrl_required(pC), "not required"); + CHECK(chk(pC, "yes") == DAS_OKAY, "yes"); + CHECK(chk(pC, "OFF") == DAS_OKAY, "OFF"); + CHECK(chk(pC, "maybe") == DASERR_SRC, "maybe"); + } + + char sBuf[64]; + p = byKey(pSrc, "formats.csv.enabled"); + CHECK(p != NULL, "formats.csv.enabled"); + if(p){ + CHECK(streq(DasSrcCtrl_param(DasSrcProp_ctrl(p)), "format.type"), "format selector param"); + const char* sTitle = DasSrcProp_title(p, sBuf, sizeof(sBuf)); + CHECK(strncmp(sTitle, "Output as ", 10) == 0, "composed format title, got '%s'", sTitle); + } + + pC = ctrlByKey(pSrc, "formats.csv.delimiter"); + CHECK(pC != NULL, "formats.csv.delimiter"); + if(pC){ + CHECK(DasSrcProp_type(DasSrcCtrl_prop(pC)) == das_srcprop_enum, "delimiter is an enum"); + CHECK(DasSrcCtrl_enumCount(pC) == 4, "four delimiters, got %zu", DasSrcCtrl_enumCount(pC)); + CHECK(DasSrcCtrl_enum(pC, 4) == NULL, "past the end"); + CHECK(streq(DasSrcCtrl_enum(pC, 0), "Comma"), "node spelling kept"); + CHECK(chk(pC, "tab") == DAS_OKAY, "enum match is case insensitive"); + CHECK(chk(pC, "colon") == DASERR_SRC, "colon is not offered"); + } + + pC = ctrlByKey(pSrc, "formats.csv.fracSecs"); + CHECK(pC != NULL, "formats.csv.fracSecs"); + if(pC){ + const char* sLo = NULL; const char* sHi = NULL; + CHECK(DasSrcProp_type(DasSrcCtrl_prop(pC)) == das_srcprop_int, "integer from the parameter"); + CHECK(DasSrcCtrl_range(pC, &sLo, &sHi), "range from the parameter"); + CHECK(streq(sLo, "0") && streq(sHi, "9"), "0 to 9, got %s to %s", sLo, sHi); + CHECK(chk(pC, "12") == DAS_OKAY, "12 passes, ranges are not enforced"); + CHECK(chk(pC, "3.5") == DASERR_SRC, "3.5 is not an integer"); + } + + /* Lookup by typed key */ + char sWhy[256]; + CHECK(DasSrc_findProp(pSrc, "time.min", sWhy, sizeof(sWhy)) == byKey(pSrc, "coords.time.min"), "time.min"); + CHECK(DasSrc_findProp(pSrc, "coord.time.minimum", NULL, 0) == byKey(pSrc, "coords.time.min"), "long form"); + CHECK(DasSrc_findProp(pSrc, "lshell", NULL, 0) == byKey(pSrc, "data.lshell.enabled"), "group toggle"); + CHECK(DasSrc_findProp(pSrc, "csv", NULL, 0) == byKey(pSrc, "formats.csv.enabled"), "format toggle"); + CHECK(DasSrc_findProp(pSrc, "serial", NULL, 0) != NULL, "unique bare prop"); + CHECK(DasSrc_findProp(pSrc, "fracSecs", sWhy, sizeof(sWhy)) == NULL, "ambiguous bare prop"); + CHECK(strstr(sWhy, "formats.das.fracSecs") && strstr(sWhy, "formats.csv.fracSecs"), "names both: %s", sWhy); + CHECK(DasSrc_findProp(pSrc, "nope", sWhy, sizeof(sWhy)) == NULL && sWhy[0], "unknown key has a reason"); + CHECK(DasSrc_findProp(pSrc, "time.units", NULL, 0) != NULL, "fixed props are found too"); + + /* Examples and requisitions */ + CHECK(DasSrc_exampleCount(pSrc) == 1, "one example"); + CHECK(streq(DasSrc_exampleLabel(pSrc, 0), "First day of 2026"), "label"); + CHECK(DasSrc_exampleLabel(pSrc, 1) == NULL, "past the examples"); + daslog_setlevel(DASLOG_CRIT); + CHECK(DasSrc_newReq(pSrc, 9) == NULL, "no example 9"); + daslog_setlevel(DASLOG_WARN); + + DasReq* pReq = DasSrc_newReq(pSrc, 0); + CHECK(pReq != NULL, "example req"); + if(pReq){ + CHECK(streq(DasReq_label(pReq), "First day of 2026"), "req carries the label"); + CHECK(streq(DasReq_srcPath(pReq), DasSrc_path(pSrc)), "stamped with the path"); + CHECK(streq(DasReq_value(pReq, "coords.time.min"), "2026-01-01"), "example setting in key form"); + CHECK(DasReq_count(pReq) == 3, "three settings, got %zu", DasReq_count(pReq)); + size_t uBad = 9; + CHECK(DasSrc_check(pSrc, pReq, &uBad) == DAS_OKAY && uBad == 0, "example checks clean"); + CHECK(DasReq_verdict(pReq, 0) == das_req_ok, "verdict ok"); + del_DasReq(pReq); + } + + pReq = DasSrc_newReq(pSrc, DASSRC_BLANK_REQ); + CHECK(pReq != NULL, "blank req"); + if(pReq){ + CHECK(DasReq_count(pReq) == 3, "lists the three required keys, got %zu", DasReq_count(pReq)); + CHECK(streq(DasReq_value(pReq, "coords.time.inter"), ""), "with empty values"); + size_t uBad = 0; + CHECK(DasSrc_check(pSrc, pReq, &uBad) == DASERR_SRC && uBad == 3, "all three bad, got %zu", uBad); + CHECK(strstr(DasReq_why(pReq, 0), "required") != NULL, "says required: %s", DasReq_why(pReq, 0)); + CHECK(DasReq_set(pReq, "time", "2026-04-01,2026-04-01T01:30") == DAS_OKAY, "range"); + CHECK(DasSrc_check(pSrc, pReq, &uBad) == DASERR_SRC && uBad == 1, "inter still bad, got %zu", uBad); + size_t uDerived = 0; + for(size_t u = 0; u < DasReq_count(pReq); ++u) if(DasReq_derived(pReq, u)) ++uDerived; + CHECK(uDerived == 2, "min and max derived, got %zu", uDerived); + DasReq_set(pReq, "inter", "60"); + CHECK(DasSrc_check(pSrc, pReq, &uBad) == DAS_OKAY, "now clean"); + char sUrl[2048]; + CHECK(DasStreamSrc_url(pSrc, pReq, 0, sUrl, sizeof(sUrl)) > 0, "builds"); + CHECK(streq(sUrl, GB "?read.time.min=2026-04-01&read.time.max=2026-04-01T01%3A30&read.time.inter=60"), + "got %s", sUrl); + size_t uNeed = DasStreamSrc_url(pSrc, pReq, 0, sUrl, 20); + CHECK(uNeed > 20 && strlen(sUrl) == 19, "snprintf style on a short buffer, need %zu", uNeed); + CHECK(DasStreamSrc_url(pSrc, pReq, 1, sUrl, sizeof(sUrl)) > 0 && + strncmp(sUrl, "http://127.0.0.1:8765/data/geomag_alt.d2s?", 42) == 0, "second mirror used"); + daslog_setlevel(DASLOG_CRIT); + CHECK(DasStreamSrc_url(pSrc, pReq, 2, sUrl, sizeof(sUrl)) == 0, "no third mirror"); + daslog_setlevel(DASLOG_WARN); + del_DasReq(pReq); + } + + /* The URL cases das_fetch has always been held to */ + URL_IS(pSrc, -1, K("time=2026-04-01,2026-04-01T01:30,60", "lshell"), NONE, + GB "?read.time.min=2026-04-01&read.time.max=2026-04-01T01%3A30&read.time.inter=60&read.data=lshell"); + URL_IS(pSrc, -1, K("coord.time.minimum=2026-02-01", "time.maximum=2026-02-02", "inter=60"), NONE, + GB "?read.time.min=2026-02-01&read.time.max=2026-02-02&read.time.inter=60"); + URL_IS(pSrc, 0, NONE, NONE, + GB "?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60"); + URL_IS(pSrc, 0, K("time.inter=30"), NONE, + GB "?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=30"); + URL_IS(pSrc, 0, K("lshell=no"), NONE, + GB "?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60"); + URL_IS(pSrc, 0, K("csv", "delimiter=Tab", "csv.headers=false", "serial=text"), NONE, + GB "?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60&format.serial=text&format.type=csv&format.headers=1&format.delim=tab"); + URL_IS(pSrc, 0, K("data.lshell", "coord.time.inter=30"), NONE, + GB "?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=30&read.data=lshell"); + const char* aRaw[] = {"read.time.inter", "5", "extra", "1", NULL}; + { + char sUrl[2048]; const char* aK[] = {NULL}; + CHECK(urlFor(pSrc, 0, aK, aRaw, sUrl, sizeof(sUrl)) && + streq(sUrl, GB "?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=5&extra=1"), + "raw parameters win and add, got %s", sUrl); + } + /* the span form */ + URL_IS(pSrc, -1, K("time=2026-01-02T14", "inter=60"), NONE, + GB "?read.time.min=2026-01-02T14%3A00%3A00&read.time.max=2026-01-02T15%3A00%3A00&read.time.inter=60"); + + URL_FAILS(pSrc, -1, K("time=2026-04-01,2026-04-01T01:30")); /* required inter missing */ + URL_FAILS(pSrc, 0, K("foo=1")); /* no such key */ + URL_FAILS(pSrc, 0, K("fracSecs=4")); /* ambiguous */ + URL_FAILS(pSrc, 0, K("formats.xml")); /* no such format */ + URL_FAILS(pSrc, 0, K("lshell=maybe")); /* not a boolean */ + URL_FAILS(pSrc, 0, K("delimiter=Colon")); /* not in enum */ + URL_FAILS(pSrc, 0, K("version=das3")); /* geomag offers only das2 */ + URL_FAILS(pSrc, 0, K("data.nope")); /* no such variable */ + URL_FAILS(pSrc, 0, K("time.units=TAI")); /* fixed property */ + URL_FAILS(pSrc, 0, K("time=2026-01-02,2026-01-03,4,5")); /* too many range items */ + + /* mime follows the format choice */ + pReq = DasSrc_newReq(pSrc, 0); + CHECK(streq(DasSrc_mime(pSrc, NULL), "application/vnd.das.stream") || + strstr(DasSrc_mime(pSrc, NULL), "das") != NULL, "default is a das stream: %s", DasSrc_mime(pSrc, NULL)); + DasReq_set(pReq, "csv", "true"); + CHECK(streq(DasSrc_mime(pSrc, pReq), "text/csv"), "csv chosen: %s", DasSrc_mime(pSrc, pReq)); + del_DasReq(pReq); + + /* A requisition for some other source is refused */ + pReq = new_DasReq(); + DasReq_setSrc(pReq, "tag:das2.org,2012:site:/elsewhere", "", ""); + DasReq_set(pReq, "time", "2026-01-02"); + daslog_setlevel(DASLOG_CRIT); + CHECK(DasSrc_check(pSrc, pReq, NULL) == DASERR_SRC, "stamp mismatch refused"); + daslog_setlevel(DASLOG_WARN); + CHECK(DasReq_verdict(pReq, 0) == das_req_unchecked, "and nothing graded"); + + /* The same URI from another mirror, or a local copy, is the same source */ + DasReq_set(pReq, "inter", "60"); + DasReq_setSrc(pReq, DasSrc_path(pSrc), "file:///somewhere/else/geomag.json", ""); + CHECK(DasSrc_check(pSrc, pReq, NULL) == DAS_OKAY, "the URL is not identity"); + DasReq_setSrc(pReq, "", "file:///somewhere/else/geomag.json", ""); + CHECK(DasSrc_check(pSrc, pReq, NULL) == DAS_OKAY, "URL only is unstamped"); + del_DasReq(pReq); + + /* A source built with a shorthand path stamps the full URI */ + size_t uLen = 0; + char* sDoc = readFile(GEOMAG, &uLen); + DasSrc* pShort = sDoc ? new_DasSrc_json(sDoc, uLen, "site:uiowa/tracers/tracers-2/spice/geomagcoords/flex", NULL) : NULL; + free(sDoc); + CHECK(pShort && streq(DasSrc_path(pShort), DasSrc_path(pSrc)), "shorthand expanded, slash supplied: %s", + pShort ? DasSrc_path(pShort) : "(null)"); + if(pShort){ + pReq = DasSrc_newReq(pShort, 0); + CHECK(pReq && (DasSrc_check(pSrc, pReq, NULL) == DAS_OKAY), "its requisitions grade here"); + del_DasReq(pReq); + del_DasSrc(pShort); + } + + del_DasSrc(pSrc); +} + +static void test_hsk(void) +{ + printf("hsk, current layout at scale:\n"); + DasSrc* pSrc = loadSrc(HSK, NULL); + CHECK(pSrc != NULL, "load"); + if(pSrc == NULL) return; + + CHECK(ctrlCount(pSrc, -1) == 90, "3 + 73 + 14 controls, got %zu", ctrlCount(pSrc, -1)); + CHECK(ctrlCount(pSrc, das_src_sec_data) == 73, "73 variable toggles"); + CHECK(DasSrc_groupCount(pSrc, das_src_sec_data) == 73, "73 variable groups"); + CHECK(DasSrc_needsAuth(pSrc), "restricted"); + CHECK(DasSrc_contactCount(pSrc) >= 1, "has a contact"); + const char* sEmail = NULL; + const char* sName = DasSrc_contact(pSrc, 0, &sEmail); + CHECK(sName && sName[0] && sEmail, "contact name %s", sName ? sName : "(none)"); + CHECK(DasSrc_contact(pSrc, 99, NULL) == NULL, "past the contacts"); + CHECK(strncmp(DasSrc_path(pSrc), "tag:das2.org,2012:local:/", 25) == 0, "no path given, so named by location: %s", + DasSrc_path(pSrc)); + CHECK(strstr(DasSrc_path(pSrc), "/" HSK) != NULL, "the location is in the name"); + + /* Every property has a key, every control a parameter, all titled */ + bool bAllKeyed = true, bAllParam = true, bAllTitled = true; + char sBuf[64]; + for(size_t u = 0; u < DasSrc_propCount(pSrc); ++u){ + const DasSrcProp* p = DasSrc_prop(pSrc, u); + if(DasSrcProp_key(p)[0] == '\0') bAllKeyed = false; + if(DasSrcProp_ctrl(p) && (DasSrcCtrl_param(DasSrcProp_ctrl(p)) == NULL)) bAllParam = false; + if(DasSrcProp_title(p, sBuf, sizeof(sBuf)) == NULL) bAllTitled = false; + } + CHECK(bAllKeyed && bAllParam && bAllTitled, "all keyed, parametered, titled"); + + /* groups are index ranges into one array, so a group's props are adjacent */ + const DasSrcGroup* pG = DasSrc_group(pSrc, das_src_sec_data, 5); + CHECK(pG && (DasSrcGroup_prop(pG, 0) == DasSrcProp_group(DasSrcGroup_prop(pG, 0))->pSrc->aProp + pG->uFirst), + "group indexing"); + + const DasSrcCtrl* pC = ctrlByKey(pSrc, "coords.time.min"); + CHECK(pC != NULL, "coords.time.min"); + if(pC){ + const char* sLo = NULL; const char* sHi = NULL; + CHECK(DasSrcCtrl_range(pC, &sLo, &sHi), "coordinate validRange reaches min"); + CHECK(sLo && (strncmp(sLo, "2025-", 5) == 0), "range low is the mission start, got %s", sLo); + CHECK(streq(DasSrcProp_title(DasSrcCtrl_prop(pC), sBuf, sizeof(sBuf)), "Minimum time value to stream"), + "node title wins over the composed one"); + CHECK(streq(DasSrcProp_units(DasSrcCtrl_prop(pC)), "UTC"), "UTC from the group units"); + } + pC = ctrlByKey(pSrc, "coords.time.res"); + CHECK(pC && !DasSrcCtrl_range(pC, NULL, NULL) && !DasSrcCtrl_required(pC), "res: no range, optional"); + const DasSrcProp* p = byKey(pSrc, "formats.csv.headers"); + CHECK(p && streq(DasSrcProp_value(p), "true") && (DasSrcProp_type(p) == das_srcprop_bool), "headers default true"); + + /* format groups: das is on by default, mime from it */ + pG = DasSrc_findGroup(pSrc, das_src_sec_format, "das"); + CHECK(pG && DasSrcGroup_mimeCount(pG) == 3, "three das mimes"); + CHECK(streq(DasSrc_mime(pSrc, NULL), "application/vnd.das2.das2stream"), "default mime: %s", DasSrc_mime(pSrc, NULL)); + + URL_IS(pSrc, 0, NONE, NONE, + HB "?read.time.min=2026-01-06T14%3A12&read.time.max=2026-01-06T14%3A24"); + URL_IS(pSrc, 0, K("version=das2", "nmi_errors", "irq3_errors", "data.hsk_task_errors"), NONE, + HB "?read.time.min=2026-01-06T14%3A12&read.time.max=2026-01-06T14%3A24&read.data=nmi_errors%2Cirq3_errors%2Chsk_task_errors&format.version=2"); + URL_IS(pSrc, 1, NONE, NONE, + HB "?read.time.min=2026-01-05T14%3A24&read.time.max=2026-01-06T14%3A24&format.type=cdf"); + URL_IS(pSrc, 0, K("formats.csv", "das.version=das2", "time.res=10"), NONE, + HB "?read.time.min=2026-01-06T14%3A12&read.time.max=2026-01-06T14%3A24&bin.time.max=10&format.version=2&format.type=csv"); + + DasReq* pReq = DasSrc_newReq(pSrc, 1); + CHECK(pReq && streq(DasSrc_mime(pSrc, pReq), "application/x-cdf"), "cdf example means a cdf: %s", + pReq ? DasSrc_mime(pSrc, pReq) : ""); + del_DasReq(pReq); + + del_DasSrc(pSrc); +} + +static void test_juno(void) +{ + printf("juno_survey, 2018 layout:\n"); + DasSrc* pSrc = loadSrc(JUNO, "tag:das2.org,2012:site:/uiowa/juno/wav/survey/das2"); + CHECK(pSrc != NULL, "load"); + if(pSrc == NULL) return; + + CHECK(ctrlCount(pSrc, das_src_sec_coord) == 3, "three time controls"); + CHECK(ctrlCount(pSrc, das_src_sec_option) >= 8, "flag options found"); + CHECK(ctrlCount(pSrc, das_src_sec_data) == 0, "nothing settable in data"); + CHECK(DasSrc_groupCount(pSrc, das_src_sec_data) > 0, "but the variables are listed"); + CHECK(DasSrc_groupCount(pSrc, das_src_sec_option) == 1, "one options group"); + CHECK(DasSrc_findGroup(pSrc, das_src_sec_option, "") != NULL, "named by the empty string"); + + /* the fixed frequency axis is visible before any download */ + const DasSrcGroup* pG = DasSrc_findGroup(pSrc, das_src_sec_coord, "frequency"); + CHECK(pG != NULL, "frequency group"); + if(pG){ + CHECK(DasSrcGroup_ctrlCount(pG) == 0, "nothing to set"); + const DasSrcProp* pU = DasSrcGroup_findProp(pG, "units"); + CHECK(pU && (DasSrcProp_ctrl(pU) == NULL) && DasSrcProp_value(pU)[0], "units %s", + pU ? DasSrcProp_value(pU) : "(none)"); + } + /* time first even though the node lists frequency first */ + CHECK(streq(DasSrcGroup_name(DasSrc_group(pSrc, das_src_sec_coord, 0)), "time"), "time first"); + + /* Keys use the node's long spelling, matching folds it */ + const DasSrcProp* p = byKey(pSrc, "coords.time.minimum"); + CHECK(p != NULL, "coords.time.minimum keyed as spelled"); + CHECK(byKey(pSrc, "coords.time.min") == NULL, "not rewritten to min"); + if(p){ + const DasSrcCtrl* pC = DasSrcProp_ctrl(p); + CHECK(pC && streq(DasSrcCtrl_param(pC), "start_time"), "param"); + CHECK(pC && DasSrcCtrl_required(pC), "required from the set block"); + CHECK(streq(DasSrcProp_value(p), "2018-10-02T11:00"), "default kept"); + const char* sLo = NULL; const char* sHi = NULL; + CHECK(pC && DasSrcCtrl_range(pC, &sLo, &sHi) && streq(sHi, "now"), "set range to now"); + CHECK(DasSrcProp_type(p) == das_srcprop_time, "type from the old parameter dict"); + CHECK(streq(DasSrcProp_units(p), "UTC"), "UTC from the old units dict"); + char sBuf[64]; + CHECK(streq(DasSrcProp_title(p, sBuf, sizeof(sBuf)), "Minimum Time value to output"), + "composed from the group name, got '%s'", DasSrcProp_title(p, sBuf, sizeof(sBuf))); + } + p = byKey(pSrc, "coords.time.resolution"); + CHECK(p && streq(DasSrcProp_value(p), "86.4"), "number default in source notation"); + CHECK(p && streq(DasSrcProp_units(p), "s"), "units string on the property"); + + p = byKey(pSrc, "options.lfrl"); + CHECK(p != NULL, "options.lfrl"); + if(p){ + CHECK(streq(DasSrcGroup_name(DasSrcProp_group(p)), ""), "options have no group name"); + CHECK(DasSrcProp_type(p) == das_srcprop_bool, "flag toggle is boolean"); + CHECK(streq(DasSrcCtrl_param(DasSrcProp_ctrl(p)), "params"), "packed parameter"); + CHECK(streq(DasSrcProp_value(p), "false"), "default false"); + } + p = byKey(pSrc, "options.das2_text"); + CHECK(p && (DasSrcProp_type(p) == das_srcprop_bool), "boolean from a boolean default"); + + /* 2018 examples come out in interface terms */ + CHECK(DasSrc_exampleCount(pSrc) >= 8, "examples, got %zu", DasSrc_exampleCount(pSrc)); + CHECK(streq(DasSrc_exampleLabel(pSrc, 0), "Perijove 16"), "label from the title: %s", DasSrc_exampleLabel(pSrc, 0)); + DasReq* pReq = DasSrc_newReq(pSrc, 0); + CHECK(pReq != NULL, "example 0 req"); + if(pReq){ + CHECK(streq(DasReq_value(pReq, "coords.time.minimum"), "2018-10-02T11:00"), "start_time mapped back"); + CHECK(streq(DasReq_value(pReq, "coords.time.resolution"), "86.4"), "resolution mapped back"); + CHECK(DasReq_rawCount(pReq) == 0, "nothing left raw, %zu", DasReq_rawCount(pReq)); + del_DasReq(pReq); + } + pReq = DasSrc_newReq(pSrc, 7); + CHECK(pReq != NULL, "example 7 req"); + if(pReq){ + CHECK(streq(DasReq_value(pReq, "options.b"), "true"), "params=B became the b toggle"); + CHECK(DasReq_rawCount(pReq) == 0, "nothing left raw"); + del_DasReq(pReq); + } + + URL_IS(pSrc, -1, K("time=2021-02-21,2021-02-22", "b", "lfrl", "das2_text"), NONE, + JB "&end_time=2021-02-22&start_time=2021-02-21¶ms=B%20LFRL&ascii=true"); + URL_IS(pSrc, -1, K("time=2021-02-21,2021-02-22,0", "ascii", "option.xmas=true"), NONE, + JB "&end_time=2021-02-22&start_time=2021-02-21&resolution=0¶ms=ASCII%20XMAS"); + /* example 7 with the resolution changed. The parameter order is the + interface order now that the example is settings, not raw text. */ + URL_IS(pSrc, 7, K("resolution=10"), NONE, + JB "&end_time=2016-177T00%3A00&start_time=2016-176T00%3A00&resolution=10¶ms=B"); + + del_DasSrc(pSrc); +} + +static void test_req(void) +{ + printf("Requisitions on their own:\n"); + DasReq* pReq = new_DasReq(); + CHECK(pReq != NULL, "new"); + CHECK(DasReq_count(pReq) == 0 && streq(DasReq_srcPath(pReq), ""), "empty and unstamped"); + daslog_setlevel(DASLOG_CRIT); + CHECK(DasReq_set(pReq, "", "x") == DASERR_SRC, "empty key refused"); + daslog_setlevel(DASLOG_WARN); + CHECK(DasReq_set(pReq, "Time", "2026-01-02") == DAS_OKAY, "set"); + CHECK(DasReq_set(pReq, "time", "2026-01-03") == DAS_OKAY, "replace, case insensitive"); + CHECK(DasReq_count(pReq) == 1 && streq(DasReq_value(pReq, "TIME"), "2026-01-03"), "one entry"); + CHECK(DasReq_set(pReq, "res", NULL) == DAS_OKAY && streq(DasReq_value(pReq, "res"), ""), "NULL is empty"); + CHECK(DasReq_find(pReq, "res") == 1 && DasReq_find(pReq, "nope") == -1, "find"); + CHECK(DasReq_verdict(pReq, 0) == das_req_unchecked && streq(DasReq_why(pReq, 0), ""), "unchecked"); + CHECK(DasReq_setRaw(pReq, "extra", "1") == DAS_OKAY && DasReq_rawCount(pReq) == 1, "raw"); + const char* sVal = NULL; + CHECK(streq(DasReq_raw(pReq, 0, &sVal), "extra") && streq(sVal, "1"), "raw read back"); + CHECK(DasReq_remove(pReq, "res") && !DasReq_remove(pReq, "res"), "remove once"); + DasReq_setLabel(pReq, "a \"quoted\" label\n"); + DasReq_setSrc(pReq, "site:/x", "http://x/y.json", "0.7"); + + /* round trip through text */ + char* sText = DasReq_text(pReq); + CHECK(sText != NULL && strstr(sText, "\\\"quoted\\\"") != NULL, "escaped label"); + DasReq* pBack = new_DasReq_text(sText, strlen(sText)); + CHECK(pBack != NULL, "parses back"); + if(pBack){ + CHECK(streq(DasReq_label(pBack), "a \"quoted\" label\n"), "label survives"); + CHECK(streq(DasReq_srcPath(pBack), "site:/x") && streq(DasReq_srcVersion(pBack), "0.7"), "stamp survives"); + CHECK(streq(DasReq_value(pBack, "time"), "2026-01-03"), "setting survives"); + CHECK(DasReq_rawCount(pBack) == 1, "raw survives"); + del_DasReq(pBack); + } + free(sText); + + /* the example settings shape reads too */ + const char* sEx = "{\"settings\":{\"coords/time/props/min\":\"2026-01-01\",\"coords/time/props/max\":\"2026-01-02\"}," + "\"label\":\"ex\"}"; + pBack = new_DasReq_text(sEx, strlen(sEx)); + CHECK(pBack && streq(DasReq_value(pBack, "coords.time.min"), "2026-01-01"), "slash paths fold to keys"); + del_DasReq(pBack); + + daslog_setlevel(DASLOG_CRIT); + CHECK(new_DasReq_text("[1,2]", 5) == NULL, "not an object"); + CHECK(new_DasReq_file("test/das_fetch/no_such_file.json") == NULL, "missing file"); + daslog_setlevel(DASLOG_WARN); + + /* file round trip and a copy */ + CHECK(DasReq_write(pReq, "tmp/TestSource_req.json") == DAS_OKAY, "write"); + pBack = new_DasReq_file("tmp/TestSource_req.json"); + CHECK(pBack && streq(DasReq_value(pBack, "time"), "2026-01-03"), "read back from file"); + del_DasReq(pBack); + remove("tmp/TestSource_req.json"); + + DasReq* pCopy = copy_DasReq(pReq); + CHECK(pCopy && DasReq_count(pCopy) == 1 && DasReq_rawCount(pCopy) == 1, "copy"); + del_DasReq(pCopy); + + DasReq_clear(pReq); + CHECK(DasReq_count(pReq) == 0 && DasReq_rawCount(pReq) == 0 && streq(DasReq_srcPath(pReq), "site:/x"), + "clear keeps the stamp"); + del_DasReq(pReq); +} + +static void test_node(void) +{ + printf("From a node tree:\n"); + DasNode* pRoot = new_RootNode_url(GEOMAG, "tag:das2.org,2012:site:/uiowa/geomag/flex", NULL, "TestSource"); + CHECK(pRoot != NULL, "node loads"); + if(pRoot == NULL) return; + CHECK(_DasSrc_defNode(pRoot) == pRoot, "a source node is its own default"); + DasSrc* pSrc = new_DasSrc_node(pRoot); + CHECK(pSrc != NULL, "source from node"); + del_RootNode(pRoot); /* the source must not care */ + if(pSrc){ + CHECK(streq(DasSrc_path(pSrc), "tag:das2.org,2012:site:/uiowa/geomag/flex"), "path from the node"); + CHECK(ctrlCount(pSrc, -1) == 17, "same interface after the tree is gone"); + URL_IS(pSrc, 0, NONE, NONE, + GB "?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60"); + del_DasSrc(pSrc); + } + + daslog_setlevel(DASLOG_CRIT); + const char* sNotSrc = "{\"type\":\"Catalog\",\"catalog\":{}}"; + CHECK(new_DasSrc_json(sNotSrc, strlen(sNotSrc), NULL, NULL) == NULL, "a catalog is not a source"); + CHECK(new_DasSrc_json("not json", 8, NULL, NULL) == NULL, "garbage is not a source"); + daslog_setlevel(DASLOG_WARN); + + /* a set block with no param is a fixed property with a warning, the + rest still builds */ + const char* sDoc = + "{\"type\":\"HttpStreamSrc\",\"protocol\":{\"baseUrls\":[\"http://x/y\"],\"httpParams\":{}}," + "\"interface\":{\"options\":{" + " \"good\":{\"value\":false,\"set\":{\"param\":\"g\",\"value\":true}}," + " \"broken\":{\"value\":false,\"set\":{\"value\":true}}," + " \"doc_only\":{\"value\":\"x\"}" + "}}}"; + daslog_setlevel(DASLOG_ERROR); + DasSrc* pSrc2 = new_DasSrc_json(sDoc, strlen(sDoc), NULL, NULL); + daslog_setlevel(DASLOG_WARN); + CHECK(pSrc2 != NULL, "builds"); + if(pSrc2){ + CHECK(streq(DasSrc_path(pSrc2), "tag:das2.org,2012:local:/memory"), "nameless and placeless: %s", + DasSrc_path(pSrc2)); + CHECK(DasSrc_propCount(pSrc2) == 3 && ctrlCount(pSrc2, -1) == 1, "three props, one control"); + CHECK(streq(DasSrcProp_key(DasSrc_prop(pSrc2, 0)), "options.good"), "the good one"); + CHECK(DasSrcProp_type(DasSrc_prop(pSrc2, 0)) == das_srcprop_bool, "boolean by default value"); + CHECK(streq(DasSrcProp_title(DasSrc_prop(pSrc2, 0), NULL, 0), ""), "untitled option is empty"); + CHECK(streq(DasSrc_mime(pSrc2, NULL), ""), "no mime stated"); + URL_IS(pSrc2, -1, K("good"), NONE, "http://x/y?g=true"); + URL_IS(pSrc2, -1, NONE, NONE, "http://x/y"); + del_DasSrc(pSrc2); + } +} + +int main(int argc, char** argv) +{ + das_init(argv[0], DASERR_DIS_EXIT, 0, DASLOG_WARN, NULL); + test_grammar(); + test_scalar(); + test_helpers(); + test_geomag(); + test_hsk(); + test_juno(); + test_req(); + test_node(); + printf("%d checks passed, %d failed\n", g_nPass, g_nFail); + return g_nFail ? 1 : 0; +} diff --git a/test/das_fetch/cat/site.json b/test/das_fetch/cat/site.json index 503247e..8dc9f3b 100644 --- a/test/das_fetch/cat/site.json +++ b/test/das_fetch/cat/site.json @@ -3,7 +3,7 @@ "version": "0.5", "name": "site", "title": "Offline test catalog, stands in for tag:das2.org,2012:site:", - "separator": "/", + "separator": ":/", "catalog": { "uiowa": { "type": "Catalog", diff --git a/test/das_fetch_test.sh b/test/das_fetch_test.sh index b2e8ff1..a28a16a 100755 --- a/test/das_fetch_test.sh +++ b/test/das_fetch_test.sh @@ -93,12 +93,13 @@ expect_url $G examples.0 -- \ expect_url $G -- \ "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" -# -r may name the root node itself, PATH is then optional -expect_url -r $G 0 -- \ - "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" - -# a named root, URI=URL: the node at URL is named URI, relative paths hang off it +# -r names the root node, URI=URL, and PATH is then optional. A bare +# location is refused: a node does not know its own name, the user says +# what the file stands for. SU="tag:das2.org,2012:site:/uiowa/tracers/tracers-2/spice/geomagcoords/flex" +HU="tag:das2.org,2012:site:/uiowa/tracers/tracers-2/l1a/cdpu/hsk/flex" +expect_rc 2 url -r $G 0 # a root needs a name +expect_rc 2 url -r =$G 0 # still no name expect_url -r $SU=$G 0 -- \ "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" expect_url -r $SU=$G $SU 0 -- \ @@ -117,16 +118,19 @@ export DASFETCH_MOUNT=$SU=$G expect_url 0 -- \ "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" unset DASFETCH_MOUNT -expect_rc 2 url -m $SU=$G -r $H 0 # a root and a mount +expect_url -m $SU=$G -r $HU=$H 0 -- \ + "$HB?read.time.min=2026-01-06T14%3A12&read.time.max=2026-01-06T14%3A24" # a root and a mount make an offline catalog +expect_url -m $SU=$G -r $HU=$H $SU 0 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" expect_rc 2 url -m site:/uiowa 0 # mount needs URI=URL expect_rc 2 url -r site:/uiowa=nourl 0 # '=' but no URL after it expect_rc 3 url -r $SU=$G site:/uiowa/juno # a named root is the whole catalog # DASFETCH_ROOT stands in for -r, and -r still wins over it -export DASFETCH_ROOT=$G +export DASFETCH_ROOT=$SU=$G expect_url 0 -- \ "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" -expect_url -r $H 0 -- \ +expect_url -r $HU=$H 0 -- \ "$HB?read.time.min=2026-01-06T14%3A12&read.time.max=2026-01-06T14%3A24" unset DASFETCH_ROOT diff --git a/utilities/das_fetch.c b/utilities/das_fetch.c index b2b5ae6..fada887 100644 --- a/utilities/das_fetch.c +++ b/utilities/das_fetch.c @@ -155,18 +155,19 @@ static void prnHelpMain(void) " -h, --help This text, or with a command, that command's help.\n" " -V, --version Print the program and library version and exit.\n" "\n" -" -r, --root ROOT Where paths start, in one of three forms.\n" +" -r, --root ROOT Where paths start, in one of two forms.\n" " A catalog URI prefix for relative paths, default\n" " '" DASCAT_DEF_PREFIX "':\n" " -r tag:das2.org,2012:test:/\n" -" A URL or file of a catalog node. It becomes the only\n" -" root, nothing outside it can be reached:\n" -" -r https://tracers-dev.physics.uiowa.edu/stream/root.json\n" -" -r ./my_catalog.json\n" -" URI=URL, the node at URL named URI, again as the only\n" -" root. Paths are relative to it unless they are full\n" -" URIs under it:\n" +" URI=URL, the node at URL declared to be the catalog\n" +" node URI. It becomes the only root: paths are\n" +" relative to it unless they are full URIs under it,\n" +" and nothing outside it can be reached. Any -m mounts\n" +" join it to make an offline catalog:\n" +" -r site:=./my_site_copy.json\n" " -r 10.1037/a0040251=https://doi.org/10.1037/a0040251/root.json\n" +" A node has no name of its own, names are composed\n" +" walking down from the root, so the URI is required.\n" " Without -r the environment variable " ROOT_ENV " is\n" " used the same way, if set.\n" " -m, --mount URI=URL\n" @@ -268,7 +269,7 @@ static void prnHelpNode(void) " " PROG " node\n" " " PROG " node uiowa/juno/wav\n" " " PROG " node -s examples,coords uiowa/tracers/tracers-2/spice/geomagcoords\n" -" " PROG " -r https://tracers-dev.physics.uiowa.edu/stream/root.json node\n" +" " PROG " -r site:/uiowa/tracers=https://tracers-dev.physics.uiowa.edu/stream/root.json node\n" "\n", stdout); } @@ -351,7 +352,7 @@ static void prnHelpQuery(bool bData) "\n" " One housekeeping value from a development server, through das3_csv:\n" "\n" -" " PROG " -r https://tracers-dev.physics.uiowa.edu/stream/root.json \\\n" +" " PROG " -r site:/uiowa/tracers=https://tracers-dev.physics.uiowa.edu/stream/root.json \\\n" " data tracers-2/ql/mag/hsk time=2026-01-01,2026-01-02 sprt | das3_csv\n" "\n" " The URL a query turns into, without running it:\n" @@ -1654,26 +1655,17 @@ static void listCatalog(const DasNode* pNode, FILE* pOut, const char* sLabel, co } } -/* What to show as a node's name, and the label for it. Under a federated - * root that is the catalog URI (label "URI:"). Under a -r URL or file root - * there is no URI: the root itself shows as "Root:" with the location given, - * its descendants as "Path:" relative to it. Catalogs get their child - * separator appended so the next step is obvious. */ +/* What to show as a node's name. Every node has a catalog URI, nodes + * loaded by location carry a local: one, see D2URI_LOCAL. Catalogs get + * their child separator appended so the next step is obvious. */ static const char* nodeUri( - const DasNode* pNode, bool bUrlRoot, const char* sFallback, char* sBuf, size_t uLen + const DasNode* pNode, const char* sFallback, char* sBuf, size_t uLen ){ const char* sUri = DasNode_pathUri(pNode); - if((sUri == NULL)||(sUri[0] == '\0')){ - if(bUrlRoot){ - strcpyz(sBuf, sFallback, uLen); - return "Root:"; - } - sUri = sFallback; - } - while((*sUri == '/')||(*sUri == ':')) ++sUri; /* child of an unnamed -r root */ + if((sUri == NULL)||(sUri[0] == '\0')) sUri = sFallback; const char* sSep = DasNode_isCatalog(pNode) ? DasNode_pathSep(pNode) : NULL; snprintf(sBuf, uLen, "%s%s", sUri, sSep ? sSep : ""); - return bUrlRoot ? "Path:" : "URI:"; + return "URI:"; } /* ************************************************************************* */ @@ -1687,8 +1679,10 @@ static const char* nodeUri( /* A -r ROOT or -m URI=URL value, or their environment stand-ins, turned into * catalog policy. URI=URL names a node: the whole catalog for -r, a mount - * on the federated catalog for -m. For -r a URL or file alone is a - * nameless root and anything else is a URI prefix; -m accepts only URI=URL. + * on the federated catalog for -m. For -r anything else is a URI prefix. + * A location alone is refused for both: nodes do not know their own + * names, so a file offered as a root with no URI would get a local: name + * and its requisitions would never match the catalog node it copies. * The URL side always has a scheme or looks like a file, which is how the * '=' that matters is found when the URI itself contains one. Command line * mistakes are reported here, the catalog logs its own refusals. False on @@ -1697,12 +1691,10 @@ static bool applyRootArg(DasCat* pCat, const char* sVal, bool bMount) { for(const char* pEq = strchr(sVal, '='); pEq != NULL; pEq = strchr(pEq + 1, '=')){ if(!das_isUrlOrFile(pEq + 1)) continue; - if(pEq == sVal){ /* "=URL", a nameless root */ - if(bMount){ - fprintf(stderr, PROG ": -m needs URI=URL, a mount without a URI has nowhere to go\n"); - return false; - } - return DasCat_setRoot(pCat, pEq + 1, NULL) == DAS_OKAY; + if(pEq == sVal){ /* "=URL", no name */ + fprintf(stderr, PROG ": %s needs URI=URL, a node is named by where it sits " + "in the catalog\n", bMount ? "-m" : "-r"); + return false; } char sUri[URI_SZ]; size_t uLen = (size_t)(pEq - sVal); @@ -1720,7 +1712,10 @@ static bool applyRootArg(DasCat* pCat, const char* sVal, bool bMount) fprintf(stderr, PROG ": the part after '=' must be a URL or a file\n"); return false; } - if(das_isUrlOrFile(sVal)) return DasCat_setRoot(pCat, sVal, NULL) == DAS_OKAY; + if(das_isUrlOrFile(sVal)){ + fprintf(stderr, PROG ": a root needs its catalog URI, -r URI=%s, see -h\n", sVal); + return false; + } return DasCat_setPrefix(pCat, sVal) == DAS_OKAY; } @@ -1781,10 +1776,15 @@ static void explainUnresolved(DasCat* pCat, const char* sPath) strcpyz(sFullUri, DasCat_lastUri(pCat), sizeof(sFullUri)); /* The common mistake with a named root is asking for something it does - not cover, say so instead of hunting for a parent */ + not cover, say so instead of hunting for a parent. A local: name is + never looked up at all, see D2URI_LOCAL. */ if(!DasCat_covers(pCat, sFullUri)){ - daslog_error_v("'%s' is outside the -r root %s, use -m to mount it on the " - "federated catalog instead", sFullUri, DasCat_rootUri(pCat)); + if(DasCat_rootUri(pCat) == NULL) + daslog_error_v("Nothing is mounted at '%s', and local: names are not looked up " + "in the federated catalog", sFullUri); + else + daslog_error_v("'%s' is outside the -r root %s, use -m to mount it on the " + "federated catalog instead", sFullUri, DasCat_rootUri(pCat)); return; } @@ -2107,9 +2107,7 @@ int main(int argc, char** argv) } const char* sFullUri = DasCat_lastUri(pCat); char sShow[1200]; - /* Under a nameless root or a path by location there is no catalog URI to show */ - bool bNoUri = ((DasCat_rootUrl(pCat) != NULL) && (DasCat_rootUri(pCat) == NULL)) || das_isUrlOrFile(sPath); - const char* sLabel = nodeUri(pNode, bNoUri, sFullUri, sShow, sizeof(sShow)); + const char* sLabel = nodeUri(pNode, sFullUri, sShow, sizeof(sShow)); int nRet = RET_OK; @@ -2168,8 +2166,7 @@ int main(int argc, char** argv) prnPara(stdout, 0, sNote); fputc('\n', stdout); char sSrcShow[1200]; - const char* sSrcLabel = nodeUri(pSrc, bNoUri, - sSrcUri ? sSrcUri : "", sSrcShow, sizeof(sSrcShow)); + const char* sSrcLabel = nodeUri(pSrc, sSrcUri ? sSrcUri : "", sSrcShow, sizeof(sSrcShow)); DasHttpSrc_describe(&src, stdout, sSrcLabel, sSrcShow); } } From e9347b277e57417e2cd2efb47bcc9d474c9a2c75 Mon Sep 17 00:00:00 2001 From: cpiker Date: Tue, 27 Oct 2026 16:03:47 -0500 Subject: [PATCH 09/18] catalog: memory root versus navigation root clarified The word root now just refers to navigation trees, DasNode constructor functions renamed for consistency Developed-with: Claude Fable 5.1 --- das3/catalog.c | 10 +-- das3/catalog.h | 2 +- das3/node.c | 58 +++++++-------- das3/node.h | 174 ++++++++++++++++++++++----------------------- das3/node_priv.h | 47 ++++++------ test/TestCatalog.c | 18 ++--- test/TestSource.c | 4 +- 7 files changed, 157 insertions(+), 156 deletions(-) diff --git a/das3/catalog.c b/das3/catalog.c index 4528dcc..38e6686 100644 --- a/das3/catalog.c +++ b/das3/catalog.c @@ -61,7 +61,7 @@ struct das_cat { size_t uRoot; /* index of the root in aMounts when isolated */ DasNode* pRoot; /* the mount root, built on first use */ - DasNode** aRoots; /* every memory root made, freed with the session */ + DasNode** aRoots; /* every node a constructor gave us, freed with the session */ size_t uRoots; size_t uRootCap; @@ -111,7 +111,7 @@ bool das_isUrlOrFile(const char* sPath) return (u > 5) && (strcmp(sPath + u - 5, ".json") == 0); } -/* Take ownership of a memory root */ +/* Take ownership of a node a constructor returned */ static DasNode* _DasCat_keep(DasCat* pThis, DasNode* pRoot) { if(pRoot == NULL) return NULL; @@ -162,7 +162,7 @@ void del_DasCat(DasCat* pThis) /* Nodes borrow the credentials manager and the cache settings, so they go * first. The policy root is in aRoots like every other root. */ - for(size_t u = 0; u < pThis->uRoots; ++u) del_RootNode(pThis->aRoots[u]); + for(size_t u = 0; u < pThis->uRoots; ++u) del_DasNode(pThis->aRoots[u]); free(pThis->aRoots); if(pThis->pMgr != NULL) del_CredMngr(pThis->pMgr); free(pThis->pCache); @@ -394,7 +394,7 @@ DasNode* DasCat_resolve(DasCat* pThis, const char* sPath) if(!bNoPath && das_isUrlOrFile(sPath)){ char sUri[600]; pThis->sLastUri = das_strdup(_das_uri_local(sPath, sUri, sizeof(sUri))); - return _DasCat_keep(pThis, _DasNode_newRootUrl( + return _DasCat_keep(pThis, _DasNode_newUrl( sPath, pThis->sLastUri, pThis->pMgr, pThis->sAgent, pThis->pCache )); } @@ -446,7 +446,7 @@ DasNode* DasCat_resolve(DasCat* pThis, const char* sPath) pThis->sLastUri = das_string("%s%s", pThis->sPrefix, sPath); if(_DasCat_unrouted(pThis, pThis->sLastUri)) return NULL; - return _DasCat_keep(pThis, _DasNode_newRoot( + return _DasCat_keep(pThis, _DasNode_new( pThis->sLastUri, pThis->pMgr, pThis->sAgent, pThis->pCache )); } diff --git a/das3/catalog.h b/das3/catalog.h index 816877e..4da6113 100644 --- a/das3/catalog.h +++ b/das3/catalog.h @@ -287,7 +287,7 @@ DAS_API const char* DasCat_prefix(const DasCat* pThis); * not match the catalog node the file was copied from; give the real URI * when it has one. * - * @param sUrl A URL or file path that new_RootNode_url() accepts + * @param sUrl A URL or file path that new_DasNode_url() accepts * @param sUri The catalog URI the node has, shorthands accepted, or NULL * to name it by location * @returns DAS_OKAY, or DASERR_CAT with the reason logged diff --git a/das3/node.c b/das3/node.c index 07cd355..6bccdc8 100644 --- a/das3/node.c +++ b/das3/node.c @@ -650,7 +650,7 @@ DasNode* DasNode_subNode( DasNode* pThis, const char* sRelPath, DasCredMngr* pMgr, const char* sAgent ); -static void del_NonRootNode(DasNode* pThis); +static void _DasNode_delTree(DasNode* pThis); /* Helper for loadSubNode and subNode @@ -866,13 +866,13 @@ static DasNode* _DasNode_loadSubNode_dasCat( } else{ /* This child can't match, stop checking this URL set*/ - del_NonRootNode(pNode); + _DasNode_delTree(pNode); pNode = NULL; } } /* Must have been a look-alike child (or some other failure), try again */ - del_NonRootNode(pNode); + _DasNode_delTree(pNode); pNode = NULL; } @@ -993,22 +993,23 @@ DasNode* DasNode_subNode( /* ************************************************************************* */ /* Selective Destruction */ -/* only deletes non-root nodes and any contained non-root nodes. Not public: - a non-root node lives in its parent's pSubNodes cache, freeing one alone - would leave a dangling entry there. Callers only ever own roots. */ -static void del_NonRootNode(DasNode* pThis) +/* Frees a node and everything below it, except nodes a constructor handed + to a caller: those are freed only by del_DasNode(). Not public: a child + lives in its parent's pSubNodes cache, freeing one alone would leave a + dangling entry there. */ +static void _DasNode_delTree(DasNode* pThis) { if(pThis == NULL) return; - if(pThis->bIsRoot) return; + if(pThis->bOwner) return; - /* Cascade deletes for non-root sub items */ + /* Cascade through the children */ if(DasNode_isCatalog(pThis)){ DasCatNode* pCat = (DasCatNode*)pThis; ptrdiff_t uNodes = DasAry_lengthIn(pCat->pSubNodes, DIM0); for(ptrdiff_t u = 0; u < uNodes; ++u){ DasNode** ppSub = (DasNode**)DasAry_getAt(pCat->pSubNodes, vtUnknown, IDX0(u)); - del_NonRootNode(*ppSub); + _DasNode_delTree(*ppSub); } dec_DasAry(pCat->pSubNodes); dec_DasAry(pCat->pSubPaths); @@ -1022,14 +1023,13 @@ static void del_NonRootNode(DasNode* pThis) free(pThis); } -/* only deletes root nodes and any contained non-root nodes */ -void del_RootNode(DasNode* pThis) +void del_DasNode(DasNode* pThis) { if(pThis == NULL) return; - if(!pThis->bIsRoot) return; + if(!pThis->bOwner) return; /* a child, its parent frees it */ - pThis->bIsRoot = false; - del_NonRootNode(pThis); + pThis->bOwner = false; /* so the tree pass takes this node too */ + _DasNode_delTree(pThis); } /* ************************************************************************* */ @@ -1092,9 +1092,9 @@ bool _das_uri_isLocal(const char* sUri) } /* ************************************************************************* */ -/* Root Node Construction */ +/* Construction */ -DasNode* _DasNode_newRootUrl( +DasNode* _DasNode_newUrl( const char* sUrl, const char* sPathUri, DasCredMngr* pMgr, const char* sAgent, const DasNodeCache* pCache ){ @@ -1105,14 +1105,14 @@ DasNode* _DasNode_newRootUrl( /* Since they want a specific URL, go for the max timeout */ DasNode* pNode = _DasNode_mkNode(sUrl, sUri, pMgr, sAgent, DASHTTP_TO_MAX, pCache); - if(pNode) pNode->bIsRoot = true; + if(pNode) pNode->bOwner = true; return pNode; } -DasNode* new_RootNode_url( +DasNode* new_DasNode_url( const char* sUrl, const char* sPathUri, DasCredMngr* pMgr, const char* sAgent ){ - return _DasNode_newRootUrl(sUrl, sPathUri, pMgr, sAgent, NULL); + return _DasNode_newUrl(sUrl, sPathUri, pMgr, sAgent, NULL); } /* Append a JSON string literal, escaping the two characters that matter */ @@ -1215,7 +1215,7 @@ DasNode* _DasNode_newMountRoot( DasNode* pRoot = _DasNode_fromJson(sJson, u, "memory:mounts", "memory:mounts", ""); free(sJson); if(pRoot != NULL){ - pRoot->bIsRoot = true; + pRoot->bOwner = true; pRoot->pMgr = pMgr; pRoot->sAgent = sAgent; pRoot->pCache = pCache; @@ -1223,7 +1223,7 @@ DasNode* _DasNode_newMountRoot( return pRoot; } -DasNode* _DasNode_newRoot( +DasNode* _DasNode_new( const char* sPathUri, DasCredMngr* pMgr, const char* sAgent, const DasNodeCache* pCache ){ @@ -1276,21 +1276,21 @@ DasNode* _DasNode_newRoot( return NULL; } - /* I'm keeping this node, so set it as a root item so that it will not be - * deleted without an explicit call to del_RootNode, then delete the rest, - * pTop will not be deleted if pTop == pNode */ - pNode->bIsRoot = true; + /* The caller gets this node, so mark it theirs before freeing the walk + * above it; _DasNode_delTree() then leaves it alone even when + * pTop == pNode */ + pNode->bOwner = true; pNode->pMgr = pMgr; /* pTop was fetched without them, the kept node hands them to whatever it loads later */ pNode->sAgent = sAgent; pNode->pCache = pCache; - del_NonRootNode(pTop); + _DasNode_delTree(pTop); return pNode; } -DasNode* new_RootNode(const char* sPathUri, DasCredMngr* pMgr, const char* sAgent) +DasNode* new_DasNode(const char* sPathUri, DasCredMngr* pMgr, const char* sAgent) { - return _DasNode_newRoot(sPathUri, pMgr, sAgent, NULL); + return _DasNode_new(sPathUri, pMgr, sAgent, NULL); } diff --git a/das3/node.h b/das3/node.h index 39b99d3..3d5b55c 100644 --- a/das3/node.h +++ b/das3/node.h @@ -33,99 +33,95 @@ extern "C" { * @par Working with nodes directly * * The node constructors in this header are what DasCat is built on. Use - * them when a program manages its own roots, for instance to hold two + * them when a program manages its own trees, for instance to hold two * unrelated catalogs at once, or to load a single isolated node from a URL * and nothing else. * - * There are two basic methods of node acquisition, finding a node by - * its path from the global root of the federated das2 catalog system: - * (ex: tag:das2.org,2012:site:/swri/mars_express/aspera3/sciels) - * or by loading an explicit URL - * (ex: http://das2.org/catalog/das/site/uiowa/cassini/rpws/survey.json). + * There are two ways to get a node. new_DasNode() finds one by its path + * URI from the top of the federated das2 catalog, for example + * tag:das2.org,2012:site:/swri/mars_express/aspera3/sciels, walking down + * from the global root, which is the the un-named node at: + * + * http://das2.org/catalog/index.json * - * When describing the das2 catalog C-API there are two meanings for the word - * Root. + * and its mirrors. new_DasNode_url() loads one from an explicit URL or + * file, for example: * - * 1) Memory Roots - A node can be the in-memory root of a tree of nodes. + * http://das2.org/catalog/das/site/uiowa/cassini/rpws/survey.json * - * 2) Global Root - Root can also refer to the global das2 top-level un-named - * node which resides at http://das2.org/catalog/index.json and its mirrors. - * - * Any node loaded using new_RootNode() or new_RootNode_url() is an *in memory* - * root node. It might not be the root of a federated catalog system, but it - * is still a top level memory item that manages a memory cache for its - * subnodes. If new_RootNode() happens to be called with a URI of 'NULL' then - * the in-memory root corresponds to the federated catalog root. The reason for - * splitting the concept of the in-memory root from the global catalog root - * is to provide for isolated catalogs that are not part of the global system. - * - * When you instantiate a node from a random URL you have to give it a name. - * Just like files on a disk, catalog nodes do not know their name, it is - * derived from the name provided by the higher level catalog entries (up to - * root, which has no name). - * - * An Example of getting a data source interface definition follows: + * and does not walk down from the global catalog root at all. + * + * Ownership follows general das2C rules: the node a constructor + * (new_DasNode, new_DasNode_url) hands you is yours, and del_DasNode() on + * it frees the whole tree loaded below it. Children fetched with + * DasNode_subNode() are managed by their parent and go with it; + * del_DasNode() ignores them. new_DasNode() discards the nodes it walked + * through on the way down, so what you hold is a tree with no parent in + * memory, whether or not it is the top of any catalog. + * + * Catalog nodes do not know their own names. Like files on a disk, a + * node's URI comes from the entries above it, up to the global root, which + * has none. So a node loaded from a URL is named by the caller, or failing + * that by its location under D2URI_LOCAL. + * + * An example of getting a data source interface definition follows: * @code{.cpp} - - // Get the site root and safely cast the output. Any particular point in - // the catalog can be a root node for the purposes of sub-item queries and - // memory management. By calling new_RootNode() the returned item is a local - // root object for memory management purposes. The higher level nodes that - // were traversed to get to the requested item are deleted from memory. - - DasNode* pRoot = new_RootNode("tag:das2.org,2012:site:/uiowa", NULL, NULL); - if(!DasNode_isCatalog(pRoot)) handle_error(); - + + // Load the U. Iowa site catalog. The nodes walked through to reach it + // are freed before this returns. + + DasNode* pSite = new_DasNode("tag:das2.org,2012:site:/uiowa", NULL, NULL); + if(!DasNode_isCatalog(pSite)) handle_error(); + // Use the site catalog to get the Cassini Saturn SLS2 data source description - + const char* sRelPath = "cassini/ephemeris/saturn_sls2"; - DasNode* pNode = DasNode_subNode(pRoot, sRelPath, NULL, NULL); - + DasNode* pNode = DasNode_subNode(pSite, sRelPath, NULL, NULL); + if(!DasNode_isStreamSrc(pNode)) handle_error(); - + // Get the HTTP GET protocol definition - + const DasJdo* pDef = DasNode_getJdo(pNode, "protocol"); if(pDef == NULL) handle_error(); - - // Freeing the root node will delete all lower nodes - - del_RootNode(pRoot); - + + // Freeing the site node frees everything loaded below it, pNode and the + // document pDef points into included + + del_DasNode(pSite); + @endcode - * + * * You now have a JSON object describing how the source is queried. The * layout of HttpStreamSrc nodes is given by the das catalog schema, see * DasNode_version() for the two layouts in the wild. - * + * * For isolated sites, or for testing purposes catalog nodes can be loaded - * directly. The following example acquires a local test root and then a - * specific test node. - * + * directly. The following example acquires a local test catalog and then a + * specific test node. + * @code - - // Assume this site uses a password since it is a testing area. Create a + + // Assume this site uses a password since it is a testing area. Create a // credentials manager and pass a basic auth hash so we aren't bothered during // testing. - + DasCredMngr* pMngr = new_CredMngr(NULL); das_credential cred; - das_cred_init(&cred, "my.domain.com", "Regression Test Catalog", NULL, + das_cred_init(&cred, "my.domain.com", "Regression Test Catalog", NULL, "VVNFUk5BTUU6UEFTU1dPUkQ="); CredMngr_addCred(pMngr, &cred); - + const char* sTestUrl = "http://my.domain.com/das2/regtest.json"; - DasNode* pRoot = new_RootNode_url(sTestUrl, "regtest", pMngr, NULL); - if(!DasNode_isCatalog(pRoot)) handle_error(); - - // At this point sub-items can be loaded from the local root as above. The - // credentials manager given to the root is inherited by the lookup. - - DasNode* pNode = DasNode_subNode(pRoot, "MarsExpress/aspera/els", NULL, NULL); - + DasNode* pTop = new_DasNode_url(sTestUrl, "regtest", pMngr, NULL); + if(!DasNode_isCatalog(pTop)) handle_error(); + + // At this point sub-items can be loaded as above. The credentials manager + // given to the top node is inherited by the lookup. + + DasNode* pNode = DasNode_subNode(pTop, "MarsExpress/aspera/els", NULL, NULL); + @endcode - * - * */ /* Defines for common path_uri's */ @@ -175,9 +171,9 @@ typedef enum das_node_type_enum { /** A das catalog node, opaque. * - * Nodes are made by the new_RootNode*() constructors and DasNode_subNode(), - * read through the accessors below, and freed by del_RootNode() on the - * root that loaded them. There is no public layout: the document, the + * Nodes are made by the new_DasNode*() constructors and DasNode_subNode(), + * read through the accessors below, and freed by del_DasNode() on the + * node a constructor returned, which takes its children with it. There is no public layout: the document, the * source URL and the path URI are reached with DasNode_getJdo(), * DasNode_srcUrl() and DasNode_pathUri(). */ @@ -191,24 +187,24 @@ typedef struct das_node DasNode; */ DAS_API const char** das_root_urls(size_t* pLen); -/** Create a new root catalog node via a path URI +/** Load a node by its catalog URI * * Get a catalog node that is not attached to any parent nodes. If the - * node acquired is a container type such as Root, Scheme, or Catalog then - * it can be used to acquire further nodes. + * node acquired is a container, see DasNode_isCatalog(), then it can be + * used to acquire further nodes. * * This function consults the distributed das2 catalog to find and load nodes. - * See new_RootNode_url() for a version that only loads a specified URL. + * See new_DasNode_url() for a version that only loads a specified URL. * * @code{.cpp} DasNode* pRoot = NULL; // Get the das2 site root catalog node - pRoot = new_RootNode(D2URI_SITES, NULL, NULL); + pRoot = new_DasNode(D2URI_SITES, NULL, NULL); // Get the U. Iowa site catalog top node - pRoot = new_RootNode(D2URI_SITE_UIOWA, NULL, NULL); + pRoot = new_DasNode(D2URI_SITE_UIOWA, NULL, NULL); @endcode * @@ -232,20 +228,20 @@ DAS_API const char** das_root_urls(size_t* pLen); * node. * * @return A new DasNode object allocated on the heap, or NULL if object - * resolution failed. Calling del_RootNode() for the returned + * resolution failed. Calling del_DasNode() for the returned * pointer will delete all sub-nodes from memory as well. * * @memberof DasNode */ -DAS_API DasNode* new_RootNode( +DAS_API DasNode* new_DasNode( const char* sPathUri, DasCredMngr* pMgr, const char* sAgent ); -/** Create a new root catalog node via direct URL +/** Load a node from a URL or file * * Get a catalog node that is not attached to any parent nodes. If the - * node acquired is a container type such as Root, Scheme, or Catalog then - * it can be used to acquire further nodes. + * node acquired is a container, see DasNode_isCatalog(), then it can be + * used to acquire further nodes. * * There are two basic methods of node acquisition, set a path URI and let the * library find the node using the built-in global catalog location, or @@ -256,13 +252,13 @@ DAS_API DasNode* new_RootNode( DasNode* pRoot = NULL; // Get a standalone local Voyager catalog root - pRoot = new_RootNode_url("http://my.site.gov/test/local_vgr.json", "vgr", NULL, NULL); + pRoot = new_DasNode_url("http://my.site.gov/test/local_vgr.json", "vgr", NULL, NULL); // Get a standalone data source description for Voyager PLS data - pRoot = new_RootNode_url("http://my.site.gov/test/local_vgr/pls.json", "vgr/pls", NULL, NULL); + pRoot = new_DasNode_url("http://my.site.gov/test/local_vgr/pls.json", "vgr/pls", NULL, NULL); // Nodes need not come from a server, a path or file:// URL reads a file - pRoot = new_RootNode_url("./local_vgr.json", "vgr", NULL, NULL); + pRoot = new_DasNode_url("./local_vgr.json", "vgr", NULL, NULL); @endcode * @@ -294,7 +290,7 @@ DAS_API DasNode* new_RootNode( * * @memberof DasNode */ -DAS_API DasNode* new_RootNode_url(const char* sUrl, const char* sPathUri, +DAS_API DasNode* new_DasNode_url(const char* sUrl, const char* sPathUri, DasCredMngr* pMgr, const char* sAgent); /** Get a das2 catalog node contained item. @@ -316,7 +312,7 @@ DAS_API DasNode* new_RootNode_url(const char* sUrl, const char* sPathUri, * * @param pMgr A credentials manager object to consult if a password is * requested. NULL means inherit the manager this node was - * loaded with, which is the one given to new_RootNode() or + * loaded with, which is the one given to new_DasNode() or * its relatives. Pass one to override it for this lookup. * * @param sAgent The user agent string to send. NULL inherits the agent this @@ -478,11 +474,13 @@ DAS_API bool DasNode_isJson(const DasNode* pThis); DAS_API const DasJdo* DasNode_getJdo(const DasNode* pThis, const char* sFragment); -/** Delete a root node freeing its memory. - * All sub-nodes will be deleted as well. +/** Free a node a constructor returned, and every node loaded below it. + * + * A child node, one from DasNode_subNode(), is ignored: it belongs to its + * parent and is freed with it. * @memberof DasNode */ -DAS_API void del_RootNode(DasNode* pNode); +DAS_API void del_DasNode(DasNode* pNode); #ifdef __cplusplus diff --git a/das3/node_priv.h b/das3/node_priv.h index 11f4f1e..7c64895 100644 --- a/das3/node_priv.h +++ b/das3/node_priv.h @@ -48,14 +48,14 @@ * child's URL and remembers the child in the parent's pSubNodes array, so * a second lookup of the same path costs nothing. * - * Ownership. Trees are owned from the top. A node with bIsRoot set is a - * memory root and del_RootNode() on it frees everything loaded below it. - * Callers never free a non-root node, it belongs to its parent's pSubNodes - * array. A memory root need not be a catalog root: _DasNode_newRoot() - * walks down from the federated roots to the requested node, marks that - * one as a root, then deletes the nodes above it (del_NonRootNode() skips - * roots). An application holding site:/uiowa/juno/wav thus owns a one - * node tree with no parent in memory. + * Ownership. Trees are owned from the top. A node with bOwner set was + * handed to a caller by a constructor: it owns everything loaded below it + * and only del_DasNode() frees it. Callers never free a child node, it + * belongs to its parent's pSubNodes array. An owner need not be the top + * of any catalog: _DasNode_new() walks down from the federated roots to + * the requested node, marks that one, then deletes the nodes above it + * (_DasNode_delTree() skips owners). An application holding + * site:/uiowa/juno/wav thus owns a one node tree with no parent in memory. * * Inherited settings. pMgr, sAgent and pCache are borrowed from whoever * made the root and copied into each child as it loads, so a DasCat's @@ -77,7 +77,8 @@ struct das_node { das_node_type_e nType; char sURL[512]; /* where this node was loaded from */ char sPath[512]; /* the node's catalog path URI, may be empty */ - bool bIsRoot; /* a memory root, owns everything loaded below it */ + bool bOwner; /* handed out by a constructor: owns the tree below + it, freed only by del_DasNode() */ void* pDom; /* the parsed document, a DasJdo for JSON nodes */ DasCredMngr* pMgr; /* borrowed from the loader, inherited by children */ const char* sAgent; /* borrowed, usually a string literal */ @@ -110,7 +111,7 @@ const char* _das_uri_local(const char* sLoc, char* sOut, size_t uOut); * own mounts, never through the federated roots. */ bool _das_uri_isLocal(const char* sUri); -/* Root constructors. The public new_RootNode() and new_RootNode_url() in +/* Constructors. The public new_DasNode() and new_DasNode_url() in * node.h are thin wrappers that pass NULL for pCache, so trees made through * them are never cached. A DasCat passes its own settings; the session is * the only place caching is configured. */ @@ -120,7 +121,7 @@ bool _das_uri_isLocal(const char* sUri); * * Each root URL is tried in turn with a short connect timeout and the whole * list is retried with longer ones before giving up, so one slow mirror - * does not stall a walk. The node found is marked as a memory root and + * does not stall a walk. The node found is marked as the caller's and * every node above it is freed, see Ownership above. Credentials are not * offered to the top level roots, only to the nodes below them, which is * where restricted catalogs live. @@ -141,16 +142,16 @@ bool _das_uri_isLocal(const char* sUri); * are trusted, inherited by the whole tree. NULL means no caching. * Borrowed. * - * @returns A new memory root, to be freed with del_RootNode(), or NULL if no - * federated root could be reached or none of them lead to sPathUri. + * @returns A node the caller owns, to be freed with del_DasNode(), or NULL + * if no federated root could be reached or none lead to sPathUri. * Failures are logged, not raised through das_error(). */ -DasNode* _DasNode_newRoot( +DasNode* _DasNode_new( const char* sPathUri, DasCredMngr* pMgr, const char* sAgent, const DasNodeCache* pCache ); -/* Load one node directly from a URL or file path and make it a memory root. +/* Load one node directly from a URL or file path for the caller to own. * * The maximum connect timeout is used since there is no alternate URL to * fall back on. @@ -161,8 +162,9 @@ DasNode* _DasNode_newRoot( * * @param sPathUri The catalog URI to record for the node, which the loader * has no way to know from the URL alone. It may be any name the - * caller likes, or "" when no URI applies, as for a nameless DasCat - * root. Children extend it using the node's own separator. + * caller likes, shorthands expanded, or NULL or "" to name the node + * by its location under D2URI_LOCAL. Children extend it using the + * node's own separator. * * @param pMgr Credentials for this fetch and for every node loaded later * from the result. May be NULL. Borrowed. @@ -173,11 +175,11 @@ DasNode* _DasNode_newRoot( * @param pCache Cache settings for the tree, NULL for no caching. * Borrowed. * - * @returns A new memory root, to be freed with del_RootNode(), or NULL if - * the document could not be read or is not a recognized node type. + * @returns A node the caller owns, to be freed with del_DasNode(), or NULL + * if the document could not be read or is not a recognized node type. * Failures are logged, not raised through das_error(). */ -DasNode* _DasNode_newRootUrl( +DasNode* _DasNode_newUrl( const char* sUrl, const char* sPathUri, DasCredMngr* pMgr, const char* sAgent, const DasNodeCache* pCache ); @@ -246,8 +248,9 @@ DasNode* _DasNode_newRootUrl( * @param pCache Cache settings for the tree, NULL for no caching. * Borrowed. * - * @returns A new memory root whose source URL reports as "memory:mounts", - * to be freed with del_RootNode(), or NULL if the arguments are bad + * @returns A new mount root, owned by the caller, whose source URL reports + * as "memory:mounts", to be freed with del_DasNode(), or NULL if the + * arguments are bad * or the document could not be allocated. Nothing is fetched at * this point; the mounted nodes load on first lookup. */ diff --git a/test/TestCatalog.c b/test/TestCatalog.c index fa9be9a..ca6dcfb 100644 --- a/test/TestCatalog.c +++ b/test/TestCatalog.c @@ -81,36 +81,36 @@ int main(int argc, char** argv) { * But were are loading it is if it was an isolated item so we can name it * whatever we want, hence doggy */ - DasNode* pCasEphem = new_RootNode_url(sUrl, "doggy", NULL, sAgent); + DasNode* pCasEphem = new_DasNode_url(sUrl, "doggy", NULL, sAgent); nTest += info_or_exit(sUrl, pCasEphem, "Cassini Saturn Ephemeris", nTest); printf("TEST %d: Deleting a single node", nTest); - del_RootNode(pCasEphem); pCasEphem = NULL; + del_DasNode(pCasEphem); pCasEphem = NULL; printf(" [OKAY]\n"); nTest += 1; sUrl = "https://das2.org/catalog/das/site/uiowa/juno/wav.json"; - DasNode* pWavCat = new_RootNode_url(sUrl, "whattie", NULL, sAgent); + DasNode* pWavCat = new_DasNode_url(sUrl, "whattie", NULL, sAgent); nTest += info_or_exit(sUrl, pWavCat, "Juno Waves Catalog", nTest); const DasNode* pSurvey = DasNode_subNode(pWavCat, "survey", NULL, sAgent); nTest += info_or_exit(sUrl, pSurvey, "Juno Waves Survey Data Source", nTest); printf("TEST %d: (Not) Deleting a non-root ", nTest); - del_RootNode((DasNode*)pSurvey); /* Erasing const for testing purposes */ + del_DasNode((DasNode*)pSurvey); /* Erasing const for testing purposes */ /* Shouldn't be done in app code */ if(DasNode_srcUrl(pSurvey)[0] != '\0') printf(" ...looks like it's still here [OKAY]\n"); else printf("... [FAILED]"); nTest += 1; printf("TEST %d: Deleting a catalog node", nTest); - del_RootNode(pWavCat); pWavCat = NULL; pSurvey = NULL; + del_DasNode(pWavCat); pWavCat = NULL; pSurvey = NULL; printf(" [OKAY]\n"); nTest += 1; /* Central root catalog*/ - DasNode* pRoot = new_RootNode(sUri, NULL, sAgent); + DasNode* pRoot = new_DasNode(sUri, NULL, sAgent); /* Can't call info or exit because root URI is null */ if(pRoot == NULL){ printf("TEST %d: Find and load root node [FAILED]\n", nTest); @@ -149,7 +149,7 @@ int main(int argc, char** argv) { /* deleting top root */ printf("TEST %d: Deleting URI based top root ", nTest); - del_RootNode(pRoot); + del_DasNode(pRoot); pRoot = NULL; pDas2 = pSpase = pCdaWeb = pWavUcalHrs = NULL; printf(" [OKAY]\n"); nTest += 1; @@ -157,7 +157,7 @@ int main(int argc, char** argv) { /* direct deep dig to ferret out memory leaks */ sUri = "tag:das2.org,2012:site:/uiowa/juno/wav/survey/das2"; - DasNode* pSurvey2 = new_RootNode(sUri, NULL, sAgent); + DasNode* pSurvey2 = new_DasNode(sUri, NULL, sAgent); nTest += info_or_exit(sUri, pSurvey2, "Waves Survey Deep Read", nTest); /* Get a Query definition */ @@ -174,7 +174,7 @@ int main(int argc, char** argv) { /* And remove it */ printf("TEST %d: Deleting deep URI direct lookup", nTest); - del_RootNode(pSurvey2); + del_DasNode(pSurvey2); printf(" [OKAY]\n"); /* TODO: Add more JSON catalog tests */ diff --git a/test/TestSource.c b/test/TestSource.c index 2920988..18d114e 100644 --- a/test/TestSource.c +++ b/test/TestSource.c @@ -746,13 +746,13 @@ static void test_req(void) static void test_node(void) { printf("From a node tree:\n"); - DasNode* pRoot = new_RootNode_url(GEOMAG, "tag:das2.org,2012:site:/uiowa/geomag/flex", NULL, "TestSource"); + DasNode* pRoot = new_DasNode_url(GEOMAG, "tag:das2.org,2012:site:/uiowa/geomag/flex", NULL, "TestSource"); CHECK(pRoot != NULL, "node loads"); if(pRoot == NULL) return; CHECK(_DasSrc_defNode(pRoot) == pRoot, "a source node is its own default"); DasSrc* pSrc = new_DasSrc_node(pRoot); CHECK(pSrc != NULL, "source from node"); - del_RootNode(pRoot); /* the source must not care */ + del_DasNode(pRoot); /* the source must not care */ if(pSrc){ CHECK(streq(DasSrc_path(pSrc), "tag:das2.org,2012:site:/uiowa/geomag/flex"), "path from the node"); CHECK(ctrlCount(pSrc, -1) == 17, "same interface after the tree is gone"); From 0b97d3b6f456130f3728980502d4026a150f9a34 Mon Sep 17 00:00:00 2001 From: cpiker Date: Tue, 27 Oct 2026 22:41:16 -0500 Subject: [PATCH 10/18] das_fetch uses library DasSrc & DasReq objects New library features: * dassrc_loadable() lists which (if any) sub-nodes of a catalog node are usable by the library. * node.c is able to upgrade some older nodes until they can be replaced. This is a general feature that may be needed even after the current deprecated set is retired * das_fetch gained the ability to select among data source nodes of a catalog for it's preferred MIME types * the library now validates that URI's actually look like uri's Developed-with: Claude Fable 5.1 --- das3/catalog.c | 18 +- das3/catalog.h | 13 +- das3/json.c | 27 +- das3/json.h | 9 +- das3/node.c | 154 +++- das3/node.h | 6 +- das3/node_priv.h | 4 +- das3/req.h | 18 +- das3/source.c | 70 +- das3/source.h | 44 +- das3/src_priv.h | 7 - test/TestSource.c | 52 +- test/das_fetch/cat/old.json | 20 + test/das_fetch/cat/twin.json | 30 + test/das_fetch_test.sh | 40 +- utilities/das_fetch.c | 1587 +++++++++------------------------- 16 files changed, 794 insertions(+), 1305 deletions(-) create mode 100644 test/das_fetch/cat/old.json create mode 100644 test/das_fetch/cat/twin.json diff --git a/das3/catalog.c b/das3/catalog.c index 38e6686..e59cd66 100644 --- a/das3/catalog.c +++ b/das3/catalog.c @@ -32,7 +32,7 @@ #include "util.h" #include "log.h" #include "node_priv.h" -#include "src_priv.h" +#include "source.h" #include "catalog.h" struct das_cat_mount { @@ -366,6 +366,17 @@ static DasNode* _DasCat_root(DasCat* pThis) aUrl[u] = pThis->aMounts[u].sUrl; aUri[u] = pThis->aMounts[u].sUri; } + /* The mount root matches children in document order, so a mount at + site:/uiowa listed before one at site:/uiowa/juno/wav would shadow + it. Longest URI first makes the order mounts were given in + irrelevant, as it should be: lists in a catalog are never + preferences. Insertion sort, there are a handful at most. */ + for(size_t i = 1; i < pThis->uMounts; ++i){ + for(size_t j = i; (j > 0) && (strlen(aUri[j]) > strlen(aUri[j-1])); --j){ + const char* s = aUri[j]; aUri[j] = aUri[j-1]; aUri[j-1] = s; + s = aUrl[j]; aUrl[j] = aUrl[j-1]; aUrl[j-1] = s; + } + } pThis->pRoot = _DasNode_newMountRoot( aUrl, aUri, pThis->uMounts, !pThis->bIsolated, pThis->pMgr, pThis->sAgent, pThis->pCache @@ -510,8 +521,5 @@ DasNode* DasCat_nearest(DasCat* pThis, const char** psRest) DasSrc* DasCat_newSrc(DasCat* pThis, const char* sPath) { DasNode* pNode = DasCat_resolve(pThis, sPath); - if(pNode == NULL) return NULL; - DasNode* pSrcNode = _DasSrc_defNode(pNode); - if(pSrcNode == NULL) return NULL; - return new_DasSrc_node(pSrcNode); + return (pNode == NULL) ? NULL : new_DasSrc_node(pNode); } diff --git a/das3/catalog.h b/das3/catalog.h index 4da6113..e32fa13 100644 --- a/das3/catalog.h +++ b/das3/catalog.h @@ -385,14 +385,11 @@ DAS_API DasNode* DasCat_nearest(DasCat* pThis, const char** psRest); /** Resolve a path and build the data source it leads to. * - * Catalog paths usually end at a Collection or SourceSet whose children - * are the actual sources, often one per server interface, and users - * rarely type the child's name. This resolves the path as - * DasCat_resolve() does and then picks: the node itself if it is - * already a stream source, else the child named "flex", which is the - * current server interface, else the first stream source child in the - * container's order. The source is built from that node with - * new_DasSrc_node(). + * Resolves the path as DasCat_resolve() does and hands the node to + * new_DasSrc_node(). The path has to end at a data source not a + * Container, and it must be a catalog node type that das2C understands. + * + * @see dassrc_loadable() * * @param sPath A catalog path in any form DasCat_resolve() accepts, * including a URL or file name. diff --git a/das3/json.c b/das3/json.c index 3db06be..10ec45f 100644 --- a/das3/json.c +++ b/das3/json.c @@ -35,6 +35,7 @@ #include #include #include /* Only needed for DasJdo_get, could do manually */ +#include #include "util.h" #include "json.h" @@ -2191,9 +2192,7 @@ void* DasJdo_writeMinified(const DasJdo* pThis, size_t *out_size) { return 0; } - size += 1; // for the '\0' null terminating character - - data = (char *)malloc(size); + data = (char *)malloc(size + 1); // plus the '\0' null terminating character if (0 == data) { // malloc failed! @@ -2212,7 +2211,7 @@ void* DasJdo_writeMinified(const DasJdo* pThis, size_t *out_size) { *data_end = '\0'; if (0 != out_size) { - *out_size = size; + *out_size = size; // the length, not counting the terminator } return data; @@ -2527,9 +2526,7 @@ void *DasJdo_writePretty(const DasJdo* pThis, const char *indent, return 0; } - size += 1; // for the '\0' null terminating character - - data = (char *)malloc(size); + data = (char *)malloc(size + 1); // plus the '\0' null terminating character if (0 == data) { // malloc failed! @@ -2548,7 +2545,7 @@ void *DasJdo_writePretty(const DasJdo* pThis, const char *indent, *data_end = '\0'; if (0 != out_size) { - *out_size = size; + *out_size = size; // the length, not counting the terminator } return data; @@ -2656,13 +2653,19 @@ const DasJdo* DasJdo_get(const DasJdo* pThis, const char* sRelPath){ size_t uLen = (*pSep == '\0') ? strlen(sRelPath) : pSep - sRelPath; - if((uLen == 0)||(uLen == 1)) return NULL; /* Invalid relative path */ + if(uLen == 0) return NULL; /* Invalid relative path */ bool bRecurse = ((*pSep != '\0') && (*(pSep+1) != '\0')); - /* Convert string to number since integers are used to lookup array values - * and can also be used to find dictionary values */ + /* A segment of digits is an index, usable for arrays and dictionaries + * alike. Only this segment counts, "0/EMAIL" is index 0 then EMAIL. */ int iItem = -1; - if( ! das_str2int(sRelPath, &iItem) ) iItem = -1; + if(uLen < 10){ + iItem = 0; + for(size_t u = 0; u < uLen; ++u){ + if(!isdigit((unsigned char)sRelPath[u])){ iItem = -1; break; } + iItem = iItem*10 + (sRelPath[u] - '0'); + } + } /* Really should avoid linear searches like this */ int i = 0; diff --git a/das3/json.h b/das3/json.h index 78caa80..e87cdef 100644 --- a/das3/json.h +++ b/das3/json.h @@ -434,10 +434,12 @@ DAS_API const das_json_ary* DasJdo_array(const DasJdo* pThis); * @param pThis The top level document object to write, all sub-objects will * be written as well. * + * @param out_size If not NULL, receives the length of the string in bytes, + * not counting the null terminator, as strlen() would report. + * * @return A new buffer containing utf-8 string data, or NULL if an error * occurred (malformed JSON input, or malloc failed), * - * The out_size parameter is optional as the utf-8 string is null terminated. * @memberof DasJdo */ DAS_API void* DasJdo_writeMinified(const DasJdo* pThis, size_t *out_size); @@ -454,8 +456,11 @@ DAS_API void* DasJdo_writeMinified(const DasJdo* pThis, size_t *out_size); * * DasJdo_writePretty performs 1 call to malloc for the entire encoding. * + * @param out_size If not NULL, receives the length of the string in bytes, + * not counting the null terminator, as strlen() would report. + * * @return NULL if an error occurred (malformed JSON input, or malloc failed). - * The out_size parameter is optional as the utf-8 string is null terminated. + * @memberof DasJdo */ DAS_API void* DasJdo_writePretty(const DasJdo* pThis, const char *indent, const char *newline, size_t *out_size); diff --git a/das3/node.c b/das3/node.c index 6bccdc8..5eafd96 100644 --- a/das3/node.c +++ b/das3/node.c @@ -15,6 +15,13 @@ * version 2.1 along with libdas2; if not, see . */ +/* Current upgrades provided here (2026-10-27) + + 1. JSON Collection objects (v0.5) are given mime type data + + no others at this time +*/ + #define _POSIX_C_SOURCE 200112L #include #include @@ -452,6 +459,128 @@ static void _DasNode_writeValidators(const char* sCache, const DasHttpResp* pRes /* Build a node from JSON text. sUrl is where it came from and sFinalUrl * the same after redirects, both for messages. sPathUri may be NULL. */ +/* ************************************************************************* */ +/* Presenting every document in the current layout + * + * Past this point the library reads one document model, the current + * catalog layout. Each entry in the table below recognizes one older + * shape by node type and version and rewrites the parsed DOM to match. + * Elements the parser did not allocate come from a side block, one per + * node, freed beside the DOM. + * + * If a document format other than JSON is ever read (Das2DSDF text, SPASE + * XML), this table is where it goes: emit JSON text in the current layout + * and run it through the parser, so nothing downstream learns a second + * model. None is written; this only says where. */ + +/* ---- 2018 Collection shim, begin. When those catalogs are regenerated + delete down to the end marker, plus the _DasNode_upgrade() call in + _DasNode_fromJson() and the pShim field: a table with no entries is + not C99, so the next translator brings the table back. The comment + above stays; it says where. ---- */ + +static bool _ieq(const char* a, const char* b) +{ + if((a == NULL)||(b == NULL)) return false; + for(; *a && *b; ++a, ++b) + if(tolower((unsigned char)*a) != tolower((unsigned char)*b)) return false; + return *a == *b; +} + +/* 2018 Collections, version 0.5 and earlier, describe each source by a + convention string instead of the media types it provides. The list + the convention implies is appended to the entry so readers need only + one rule. Delete this entry when those catalogs are regenerated. */ + +struct provides_patch { /* "provides": ["mime"], all in one block */ + das_json_dict_el el; + das_json_str name; + DasJdo aryVal; + das_json_ary ary; + das_json_ary_el aryEl; + DasJdo strVal; + das_json_str mime; +}; + +static const char* _impliedProvides(const DasJdo* pEntry) +{ + if((pEntry == NULL)||(pEntry->type != das_json_type_dict)) return NULL; + if(!_ieq(DasJdo_string(DasJdo_get(pEntry, D2FRAG_TYPE)), D2CV_TYPE_STREAM)) return NULL; + if(DasJdo_get(pEntry, "provides") != NULL) return NULL; + const char* sConv = DasJdo_string(DasJdo_get(pEntry, "convention")); + if(_ieq(sConv, "das2/2.2")) return "application/vnd.das2.das2stream"; + if(_ieq(sConv, "hapi/2.0")) return "application/vnd.hapi.stream"; + return NULL; +} + +/* TODO: Remove this particular upgrade function once no old style collection + catalog nodes exist in the wild */ +static bool _upgradeCollection(DasJdo* pRoot, void** ppShim) +{ + const DasJdo* pSrcs = DasJdo_get(pRoot, D2FRAG_SOURCES); + if((pSrcs == NULL)||(pSrcs->type != das_json_type_dict)) return true; /* reported later */ + + size_t uNeed = 0; + const das_json_dict_el* pEl = NULL; + for(pEl = DasJdo_dictFirst(pSrcs); pEl; pEl = pEl->next) + if(_impliedProvides(pEl->value) != NULL) ++uNeed; + if(uNeed == 0) return true; + + struct provides_patch* aPatch = (struct provides_patch*)calloc( + uNeed, sizeof(struct provides_patch) + ); + if(aPatch == NULL) return false; + *ppShim = aPatch; + + size_t u = 0; + for(pEl = DasJdo_dictFirst(pSrcs); pEl; pEl = pEl->next){ + const char* sMime = _impliedProvides(pEl->value); + if(sMime == NULL) continue; + struct provides_patch* p = aPatch + u++; + p->name.string = "provides"; p->name.string_size = 8; + p->mime.string = sMime; p->mime.string_size = strlen(sMime); + p->strVal.type = das_json_type_str; p->strVal.value = &p->mime; + p->aryEl.value = &p->strVal; + p->ary.start = &p->aryEl; p->ary.length = 1; + p->aryVal.type = das_json_type_ary; p->aryVal.value = &p->ary; + p->el.name = &p->name; p->el.value = &p->aryVal; + + das_json_dict* pDict = (das_json_dict*)pEl->value->value; + if(pDict->start == NULL){ + pDict->start = &p->el; + } + else{ + das_json_dict_el* pLast = pDict->start; + while(pLast->next != NULL) pLast = pLast->next; + pLast->next = &p->el; + } + pDict->length += 1; + } + return true; +} + +static const struct { + das_node_type_e nType; + double rThrough; /* versions up to this one */ + bool (*fn)(DasJdo* pRoot, void** ppShim); +} g_upgrades[] = { + { d2node_collection, 0.5, _upgradeCollection }, +}; + +/* False only when memory ran out. An unversioned document counts as the + oldest, since versions were added after these layouts. */ +static bool _DasNode_upgrade(DasJdo* pRoot, das_node_type_e nType, void** ppShim) +{ + const char* sVer = DasJdo_string(DasJdo_get(pRoot, D2FRAG_VERSION)); + double rVer = (sVer == NULL) ? 0.0 : das_strtod_c(sVer, NULL); + for(size_t u = 0; u < sizeof(g_upgrades)/sizeof(g_upgrades[0]); ++u){ + if((g_upgrades[u].nType != nType)||(rVer > g_upgrades[u].rThrough)) continue; + if(!g_upgrades[u].fn(pRoot, ppShim)) return false; + } + return true; +} +/* ---- 2018 Collection shim, end ---- */ + static DasNode* _DasNode_fromJson( const char* pBytes, size_t uBytes, const char* sUrl, const char* sFinalUrl, const char* sPathUri @@ -490,13 +619,21 @@ static DasNode* _DasNode_fromJson( return NULL; } + /* Older layouts are rewritten here so the rest of the node code, and + everything built on it, reads only the current one. */ + void* pShim = NULL; + if(!_DasNode_upgrade(pRoot, nType, &pShim)){ + free(pRoot); + return NULL; + } + /* Initialize non-container types */ DasNode* pBase = NULL; if((nType == d2node_stream_src)||(nType == d2node_file_agg)){ pBase = (DasNode*)calloc(1, sizeof(DasNode)); if(pBase == NULL){ das_error(DASERR_NODE, "Couldn't allocate node"); - free(pRoot); + free(pRoot); free(pShim); return NULL; } } @@ -512,14 +649,14 @@ static DasNode* _DasNode_fromJson( (pVal->type != das_json_type_dict) ){ daslog_error_v("Error in catalog object from %s, missing dictionary " "element '%s'", sFinalUrl, sContainer); - free(pRoot); + free(pRoot); free(pShim); return NULL; } DasCatNode* pThis = (DasCatNode*)calloc(1, sizeof(DasCatNode)); if(pThis == NULL){ das_error(DASERR_NODE, "Couldn't allocate catalog node"); - free(pRoot); + free(pRoot); free(pShim); return NULL; } pBase = (DasNode*)pThis; @@ -552,6 +689,7 @@ static DasNode* _DasNode_fromJson( pBase->nType = nType; strncpy(pBase->sURL, sUrl, 511); pBase->pDom = pRoot; + pBase->pShim = pShim; if(sPathUri != NULL) strncpy(pBase->sPath, sPathUri, 511); return pBase; @@ -1014,12 +1152,10 @@ static void _DasNode_delTree(DasNode* pThis) dec_DasAry(pCat->pSubNodes); dec_DasAry(pCat->pSubPaths); } - /* Now del self */ - if(!DasNode_isJson(pThis)) - das_error(DASERR_NODE, "Memory Leak! Handling for non-JSON nodes has " - "not been implemented"); - else - free(pThis->pDom); + /* Every node holds a JSON DOM, see _DasNode_upgrade(); the shim holds + whatever was added to it on load. */ + free(pThis->pDom); + free(pThis->pShim); free(pThis); } diff --git a/das3/node.h b/das3/node.h index 3d5b55c..52dd590 100644 --- a/das3/node.h +++ b/das3/node.h @@ -62,7 +62,7 @@ extern "C" { * Catalog nodes do not know their own names. Like files on a disk, a * node's URI comes from the entries above it, up to the global root, which * has none. So a node loaded from a URL is named by the caller, or failing - * that by its location under D2URI_LOCAL. + * that, by its location under D2URI_LOCAL. * * An example of getting a data source interface definition follows: * @code{.cpp} @@ -173,9 +173,7 @@ typedef enum das_node_type_enum { * * Nodes are made by the new_DasNode*() constructors and DasNode_subNode(), * read through the accessors below, and freed by del_DasNode() on the - * node a constructor returned, which takes its children with it. There is no public layout: the document, the - * source URL and the path URI are reached with DasNode_getJdo(), - * DasNode_srcUrl() and DasNode_pathUri(). + * node a constructor returned, which takes its sub-nodes with it. */ typedef struct das_node DasNode; diff --git a/das3/node_priv.h b/das3/node_priv.h index 7c64895..1ea19d1 100644 --- a/das3/node_priv.h +++ b/das3/node_priv.h @@ -79,7 +79,9 @@ struct das_node { char sPath[512]; /* the node's catalog path URI, may be empty */ bool bOwner; /* handed out by a constructor: owns the tree below it, freed only by del_DasNode() */ - void* pDom; /* the parsed document, a DasJdo for JSON nodes */ + void* pDom; /* the parsed document, a DasJdo, current layout */ + void* pShim; /* elements added on load that the parser did not + allocate, see _DasNode_upgrade(). May be NULL */ DasCredMngr* pMgr; /* borrowed from the loader, inherited by children */ const char* sAgent; /* borrowed, usually a string literal */ const DasNodeCache* pCache; /* borrowed, inherited by children */ diff --git a/das3/req.h b/das3/req.h index 641c922..04e3549 100644 --- a/das3/req.h +++ b/das3/req.h @@ -35,14 +35,13 @@ extern "C" { * A requisition is the list of settings a client wants from a data * source. It is the form an application fills in, and it is deliberately * simple. A DasReq holds strings naming entries in the interface section - * of a data source node and nothing else: no hidden pointers, no parsed - * values and no protocol parameters. Everything that interprets a - * requisition is a function of the DasSrc it is handed to, see - * DasSrc_check() and DasStreamSrc_url() in source.h. + * of a data source node. Everything that interprets a requisition is a + * function of the DasSrc or it's derived classes. + * @see DasSrc_check() and DasStreamSrc_url() * - * That split lets a requisition outlive the interface it was built from. - * It can be saved to a file one day, reloaded the next, and used to query - * the data source again. + * A requisition is self contained and may outlive the interface it was + * built from. It can be saved to a file one day, reloaded the next, and + * used to query the data source again. * * @par Entries and verdicts * @@ -90,8 +89,9 @@ extern "C" { * the URL records where that copy of the node came from. The version is * the node format version. * - * @todo Issue #42 tracks a content version. Until the catalog spec adds - * one, a stale requisition shows up as unknown keys at check time. + * @todo The catalog spec has no version for a source's interface, only + * for the node format. Until it does, a stale requisition may contain + * unknown keys at check time. */ /** How an entry fared the last time a source looked at it */ diff --git a/das3/source.c b/das3/source.c index 99fae6d..f9bf3d7 100644 --- a/das3/source.c +++ b/das3/source.c @@ -771,7 +771,15 @@ DasSrc* new_DasSrc_node(const DasNode* pNode) { if(pNode == NULL) return NULL; if(!DasNode_isStreamSrc(pNode)){ - daslog_error_v("Node %s is not a stream source", DasNode_srcUrl(pNode)); + /* Name the children so a caller can pick one; picking is not done + here because the library has no basis for the choice. */ + const char* aNames[8]; + size_t uKids = dassrc_loadable(pNode, aNames, 8); + char sMsg[512]; + size_t n = snprintf(sMsg, sizeof(sMsg), "Node %s is not a data source", DasNode_srcUrl(pNode)); + if(uKids > 0) n = _catf(sMsg, sizeof(sMsg), n, ", its loadable children are:"); + for(size_t u = 0; (u < uKids) && (u < 8); ++u) n = _catf(sMsg, sizeof(sMsg), n, " %s", aNames[u]); + daslog_error_v("%s", sMsg); return NULL; } const DasJdo* pDom = DasNode_getJdo(pNode, NULL); @@ -786,8 +794,7 @@ DasSrc* new_DasSrc_node(const DasNode* pNode) daslog_error_v("Couldn't copy the document of node %s", DasNode_srcUrl(pNode)); return NULL; } - /* the reported size counts the terminator, which the parser rejects */ - DasSrc* pThis = new_DasSrc_json(sText, strlen(sText), DasNode_pathUri(pNode), DasNode_srcUrl(pNode)); + DasSrc* pThis = new_DasSrc_json(sText, uLen, DasNode_pathUri(pNode), DasNode_srcUrl(pNode)); free(sText); return pThis; } @@ -1599,14 +1606,13 @@ size_t DasStreamSrc_url(const DasSrc* pThis, DasReq* pReq, size_t uBase, char* s return 0; } + /* Every reason is logged, not just the first: a blank requisition lists + the required keys up front, so the first bad entry is often "min is + required" while the user's typo sits further down */ size_t uBad = 0; if(DasSrc_check(pThis, pReq, &uBad) != DAS_OKAY){ - for(size_t u = 0; u < pReq->uEntry; ++u){ - if(pReq->aEntry[u].verdict == das_req_bad){ - daslog_error_v("%s", pReq->aEntry[u].sWhy); - break; - } - } + for(size_t u = 0; u < pReq->uEntry; ++u) + if(pReq->aEntry[u].verdict == das_req_bad) daslog_error_v("%s", pReq->aEntry[u].sWhy); return 0; } @@ -1682,46 +1688,18 @@ size_t DasStreamSrc_url(const DasSrc* pThis, DasReq* pReq, size_t uBase, char* s } /* ************************************************************************* */ -/* The default source under a node, used by DasCat_newSrc() */ +/* ************************************************************************* */ +/* The sources under a container */ -DasNode* _DasSrc_defNode(DasNode* pNode) +size_t dassrc_loadable(const DasNode* pNode, const char** aNames, size_t uMax) { - if(DasNode_isStreamSrc(pNode)) return pNode; - if(!DasNode_isCatalog(pNode)){ - daslog_error_v("Node %s is not a data source", DasNode_srcUrl(pNode)); - return NULL; - } - + if((pNode == NULL) || !DasNode_isCatalog(pNode)) return 0; size_t uKids = DasNode_subCount(pNode); - const char* sPick = NULL; - bool bHasCatalogs = false; + size_t uFound = 0; for(size_t u = 0; u < uKids; ++u){ - const char* sName = DasNode_subName(pNode, u); - das_node_type_e nType = DasNode_subType(pNode, u); - if((nType == d2node_catalog)||(nType == d2node_spdf_cat)||(nType == d2node_spase_cat)) - bHasCatalogs = true; - if(nType != d2node_stream_src) continue; - if(_strieq(sName, "flex")){ sPick = sName; break; } - if(sPick == NULL) sPick = sName; - } - if(sPick == NULL){ - if(bHasCatalogs) - daslog_error_v("%s is a catalog, not a data source, pick one of its children", - DasNode_srcUrl(pNode)); - else - daslog_error_v("No stream source children under %s", DasNode_srcUrl(pNode)); - return NULL; - } - - /* Credentials, agent and cache are inherited from the node's root */ - DasNode* pSub = DasNode_subNode(pNode, sPick, NULL, NULL); - if(pSub == NULL){ - daslog_error_v("Couldn't load source '%s' from %s", sPick, DasNode_srcUrl(pNode)); - return NULL; - } - if(!DasNode_isStreamSrc(pSub)){ - daslog_error_v("Child '%s' of %s loaded but is not a stream source", sPick, DasNode_srcUrl(pNode)); - return NULL; + if(DasNode_subType(pNode, u) != d2node_stream_src) continue; + if(uFound < uMax) aNames[uFound] = DasNode_subName(pNode, u); + ++uFound; } - return pSub; + return uFound; } diff --git a/das3/source.h b/das3/source.h index db508d6..82cc722 100644 --- a/das3/source.h +++ b/das3/source.h @@ -205,18 +205,44 @@ typedef struct das_src_ctrl DasSrcCtrl; /* ************************************************************************* */ /* Making and identifying sources */ +/** The children of a catalog node that new_DasSrc_node() can load + * + * Catalog paths provided by application programs usually end at a + * Collection or SourceSet whose children are the actual data sources. + * Multiple sources are listed because different clients understand how + * to fully utilize different data source types. This is the set of + * them this library interacts with, HttpStreamSrc nodes today. Which + * of several to use is the application's decision, made on whatever + * criteria it has, such as the media types each child says it provides. + * + * @param pNode A container node. A source node itself has no children + * and reports 0. A container that only lists other containers + * also reports 0. + * + * @param aNames Receives pointers to the child names, borrowed from the + * node. May be NULL with uMax 0 to just count. + * + * @param uMax The size of aNames. + * + * @returns The number of loadable children, which may exceed uMax. + */ +DAS_API size_t dassrc_loadable(const DasNode* pNode, const char** aNames, size_t uMax); + /** Build a DasSrc from a DasNode. * * The node's document is copied, so the node and its tree may be freed - * afterwards. Usually reached through DasCat_newSrc(), which also - * picks the right child of a container node. + * afterwards. Usually reached through DasCat_newSrc(). + * + * The node must be a source itself. A container is refused with its + * loadable children named in the log, see dassrc_loadable(). * * @param pNode A node of type HttpStreamSrc. Borrowed. + * * @returns A new source, to be freed with del_DasSrc(), or NULL after - * logging why the node does not describe a source. A source with - * a thin or broken interface still builds, so that it can at - * least be listed; DasStreamSrc_url() is where missing protocol - * pieces are reported. + * logging why the node does not describe a source. A source + * with a thin or broken interface still builds, so that it can + * at least be listed; missing protocol pieces are reported by + * whichever function builds a request from it. */ DAS_API DasSrc* new_DasSrc_node(const DasNode* pNode); @@ -557,8 +583,8 @@ DAS_API void DasSrc_stamp(const DasSrc* pThis, DasReq* pReq); * * Derived entries from an earlier check are dropped first, so checking * is idempotent. Nothing is logged for bad entries, the verdicts carry - * the text; DasStreamSrc_url() logs the first one when it refuses to - * build. + * the text; the function that builds a request from them reports them + * when it refuses. * * @param puBad If not NULL, receives the number of bad entries. * @returns DAS_OKAY when every entry is good, DASERR_SRC when any is bad @@ -616,7 +642,7 @@ DAS_API const char* DasStreamSrc_baseUrl(const DasSrc* pThis, size_t uIdx); * so far, 4096 is safe. * @param uLen The size of sBuf. * @returns The length of the URL, or 0 after logging why none could be - * made: a bad entry, a mismatched stamp, a missing required + * made: every bad entry, a mismatched stamp, a missing required * parameter no key supplies, a bad base index, or a source with * no base URLs. A URL is never empty, so 0 is unambiguous. * @memberof DasSrc diff --git a/das3/src_priv.h b/das3/src_priv.h index c0bebed..524b923 100644 --- a/das3/src_priv.h +++ b/das3/src_priv.h @@ -153,11 +153,4 @@ int _DasSrc_match(const DasSrc* pThis, const char* sKey, const DasSrcProp** aOut * return that group, else NULL. How a check recognizes the range forms. */ const DasSrcGroup* _DasSrc_coordGroup(const DasSrc* pThis, const char* sKey); -/* The default source under a container node: the node itself if it is a - * stream source, else the child named "flex", else the first stream - * source child. Loads the child through the node's own tree, so - * credentials and cache settings follow. Returns NULL after logging - * when there is none. Lives in source.c, used by DasCat_newSrc(). */ -DasNode* _DasSrc_defNode(DasNode* pNode); - #endif /* _das_src_priv_h_ */ diff --git a/test/TestSource.c b/test/TestSource.c index 18d114e..f349fcc 100644 --- a/test/TestSource.c +++ b/test/TestSource.c @@ -191,12 +191,14 @@ static void test_grammar(void) static void test_scalar(void) { printf("JSON scalars:\n"); - /* Two letter keys: DasJdo_get() does not resolve one character names */ const char* sDoc = "{\"st\":\"x\",\"nm\":86.4,\"in\":42,\"tr\":true,\"fa\":false," - "\"nl\":null,\"ar\":[1],\"di\":{}}"; + "\"nl\":null,\"ar\":[1],\"di\":{},\"x\":\"one\",\"ls\":[[7]]}"; DasJdo* pDom = das_json_parse(sDoc, strlen(sDoc)); CHECK(pDom != NULL, "parse"); char sBuf[32]; + CHECK(_das_json_scalar(DasJdo_get(pDom, "x"), sBuf, 32) && streq(sBuf, "one"), "one character key"); + CHECK(_das_json_scalar(DasJdo_get(pDom, "ls/0/0"), sBuf, 32) && streq(sBuf, "7"), "index then index"); + CHECK(DasJdo_get(pDom, "12/x") == NULL, "index past the end"); CHECK(_das_json_scalar(DasJdo_get(pDom, "st"), sBuf, 32) && streq(sBuf, "x"), "string"); CHECK(_das_json_scalar(DasJdo_get(pDom, "nm"), sBuf, 32) && streq(sBuf, "86.4"), "number text kept"); CHECK(_das_json_scalar(DasJdo_get(pDom, "in"), sBuf, 32) && streq(sBuf, "42"), "integer"); @@ -749,7 +751,6 @@ static void test_node(void) DasNode* pRoot = new_DasNode_url(GEOMAG, "tag:das2.org,2012:site:/uiowa/geomag/flex", NULL, "TestSource"); CHECK(pRoot != NULL, "node loads"); if(pRoot == NULL) return; - CHECK(_DasSrc_defNode(pRoot) == pRoot, "a source node is its own default"); DasSrc* pSrc = new_DasSrc_node(pRoot); CHECK(pSrc != NULL, "source from node"); del_DasNode(pRoot); /* the source must not care */ @@ -794,6 +795,50 @@ static void test_node(void) } } +/* Choosing among a container's sources by what the client reads */ +static void test_pick(void) +{ + printf("Loadable sources under a container:\n"); + const char* aNames[4]; + DasNode* pTwin = new_DasNode_url("test/das_fetch/cat/twin.json", "site:/uiowa/twin", NULL, "TestSource"); + CHECK(pTwin != NULL, "twin loads"); + if(pTwin == NULL) return; + CHECK(dassrc_loadable(pTwin, aNames, 4) == 2 && streq(aNames[0], "flex") && streq(aNames[1], "das3"), + "two loadable children in catalog order, the DSDF is skipped"); + CHECK(dassrc_loadable(pTwin, NULL, 0) == 2, "counting only"); + daslog_setlevel(DASLOG_CRIT); + CHECK(new_DasSrc_node(pTwin) == NULL, "a container is not a source, no child is picked"); + daslog_setlevel(DASLOG_WARN); + + DasNode* pSub = DasNode_subNode(pTwin, "das3", NULL, NULL); + CHECK(pSub != NULL, "the caller's choice loads through the tree"); + CHECK(dassrc_loadable(pSub, NULL, 0) == 0, "a source has no loadable children"); + DasSrc* pSrc = (pSub == NULL) ? NULL : new_DasSrc_node(pSub); + CHECK(pSrc != NULL && streq(DasSrc_path(pSrc), "tag:das2.org,2012:site:/uiowa/twin/das3"), "source from the chosen child"); + del_DasSrc(pSrc); + del_DasNode(pTwin); + + DasNode* pOld = new_DasNode_url("test/das_fetch/cat/old.json", "site:/uiowa/old", NULL, "TestSource"); + CHECK(pOld != NULL, "old loads"); + if(pOld == NULL) return; + CHECK(dassrc_loadable(pOld, aNames, 4) == 2, "a 2018 Collection lists both HttpStreamSrc children"); + + /* Presented in the current layout: the provides list each convention + implies is there, the convention is untouched, and the DOM still + serializes with the additions in place */ + CHECK(streq(DasNode_subName(pOld, 0), "das2") && streq(DasNode_subName(pOld, 1), "hapi"), "fixture order"); + const char* sMime = DasJdo_string(DasJdo_get(DasNode_subStub(pOld, 0), "provides/0")); + CHECK(sMime != NULL && streq(sMime, "application/vnd.das2.das2stream"), "das2/2.2 presents as a das2 stream"); + sMime = DasJdo_string(DasJdo_get(DasNode_subStub(pOld, 1), "provides/0")); + CHECK(sMime != NULL && streq(sMime, "application/vnd.hapi.stream"), "hapi/2.0 presents as a hapi stream"); + CHECK(DasJdo_get(DasNode_subStub(pOld, 1), "convention") != NULL, "the convention stays"); + size_t uDoc = 0; + char* sDoc = (char*)DasJdo_writeMinified(DasNode_getJdo(pOld, NULL), &uDoc); + CHECK(sDoc != NULL && strstr(sDoc, "\"provides\":[\"application/vnd.hapi.stream\"]") != NULL, "the added list serializes"); + free(sDoc); + del_DasNode(pOld); +} + int main(int argc, char** argv) { das_init(argv[0], DASERR_DIS_EXIT, 0, DASLOG_WARN, NULL); @@ -805,6 +850,7 @@ int main(int argc, char** argv) test_juno(); test_req(); test_node(); + test_pick(); printf("%d checks passed, %d failed\n", g_nPass, g_nFail); return g_nFail ? 1 : 0; } diff --git a/test/das_fetch/cat/old.json b/test/das_fetch/cat/old.json new file mode 100644 index 0000000..1ac1ac6 --- /dev/null +++ b/test/das_fetch/cat/old.json @@ -0,0 +1,20 @@ +{ + "type": "Collection", + "version": "0.5", + "name": "old", + "title": "A 2018 layout container, entries say convention not provides", + "sources": { + "das2": { + "type": "HttpStreamSrc", + "convention": "das2/2.2", + "name": "Das2/2.2 Source", + "urls": ["test/das_fetch/juno_survey_0.6.json"] + }, + "hapi": { + "type": "HttpStreamSrc", + "convention": "hapi/2.0", + "name": "HAPI/2.0 Source", + "urls": ["test/das_fetch/juno_survey_0.6.json"] + } + } +} diff --git a/test/das_fetch/cat/twin.json b/test/das_fetch/cat/twin.json new file mode 100644 index 0000000..b410cb2 --- /dev/null +++ b/test/das_fetch/cat/twin.json @@ -0,0 +1,30 @@ +{ + "type": "SourceSet", + "version": "0.5", + "name": "twin", + "title": "Two interfaces for one dataset, offline test copy", + "separator": "/", + "catalog": { + "flex": { + "type": "HttpStreamSrc", + "label": "dasflex server", + "purpose": "primary-stream", + "provides": ["application/vnd.das2.das2stream", "text/csv", "application/x-cdf"], + "urls": ["test/das_fetch/geomag_flex_0.7.json"] + }, + "das2": { + "type": "Das2DSDF", + "label": "Das2 DSDF, a type this library does not load", + "purpose": "primary-stream", + "provides": ["application/vnd.das2.das2stream"], + "urls": ["test/das_fetch/does_not_exist.d2t"] + }, + "das3": { + "type": "HttpStreamSrc", + "label": "das3 only server", + "purpose": "primary-stream", + "provides": ["application/vnd.das.stream"], + "urls": ["test/das_fetch/hsk_flex_0.7.json"] + } + } +} diff --git a/test/das_fetch_test.sh b/test/das_fetch_test.sh index a28a16a..bc878fb 100755 --- a/test/das_fetch_test.sh +++ b/test/das_fetch_test.sh @@ -161,10 +161,46 @@ expect_url $J time=2021-02-21,2021-02-22 b lfrl das2_text -- \ expect_url $J time=2021-02-21,2021-02-22,0 ascii option.xmas=true -- \ "$JB&end_time=2021-02-22&start_time=2021-02-21&resolution=0¶ms=ASCII%20XMAS" -# a 0.6 example is raw parameters; user keys still win, required defaults +# a 0.6 example is raw parameters, read back into interface terms, so the +# URL comes out in interface order; user keys still win, required defaults # must not clobber the example's values expect_url $J 7 resolution=10 -- \ - "$JB&end_time=2016-177T00%3A00¶ms=B&resolution=10&start_time=2016-176T00%3A00" + "$JB&end_time=2016-177T00%3A00&start_time=2016-176T00%3A00&resolution=10¶ms=B" + +# --- choosing among a node's sources by -a --------------------------------- + +# twin.json offers flex (das2 stream, csv, cdf) and das3 (das3 stream); with +# no -a das_fetch asks for das3 streams, then anything. An -a list is a set. +T=$FX/cat/twin.json +O=$FX/cat/old.json +expect_url -r site:/uiowa/twin=$T 0 -- \ + "$HB?read.time.min=2026-01-06T14%3A12&read.time.max=2026-01-06T14%3A24" +expect_url -r site:/uiowa/twin=$T -a text/csv 0 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" +expect_url -r site:/uiowa/twin=$T -a 'image/png, application/x-cdf' 0 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" +expect_rc 3 url -r site:/uiowa/twin=$T -a '*/*' 0 # two sources qualify, name one +expect_rc 3 url -r site:/uiowa/twin=$T -a 'application/vnd.das.stream, text/csv' 0 # order does not pick +expect_rc 3 url -r site:/uiowa/twin=$T -a image/png 0 # none does +expect_url -r site:/uiowa/twin=$T site:/uiowa/twin/flex -a '*/*' 0 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" +if $FETCH node -r site:/uiowa/twin=$T -a '*/*' 2>/dev/null | grep -q "^ das3 "; then + nPass=$((nPass + 1)) +else + echo " FAIL: das_fetch node with an ambiguous pick did not list the children"; nFail=$((nFail + 1)) +fi + +# a 2018 Collection has no provides lists, convention stands in +expect_url -r site:/uiowa/old=$O -a application/vnd.das2.das2stream time=2021-02-21,2021-02-22 -- \ + "$JB&end_time=2021-02-22&start_time=2021-02-21" +expect_rc 3 url -r site:/uiowa/old=$O time=2021-02-21,2021-02-22 # nothing provides das3, so anything: both qualify +expect_rc 3 url -r site:/uiowa/old=$O -a text/csv time=2021-02-21,2021-02-22 + +# -m may be repeated, and the deeper mount wins whatever the order +expect_url -m site:/uiowa/twin=$T -m site:/uiowa/twin/flex=$G -a '*/*' site:/uiowa/twin/flex 0 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" +expect_url -m site:/uiowa/twin/flex=$G -m site:/uiowa/twin=$T -a '*/*' site:/uiowa/twin/flex 0 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" # --- error paths, by exit code ---------------------------------------------- diff --git a/utilities/das_fetch.c b/utilities/das_fetch.c index fada887..33ad4ae 100644 --- a/utilities/das_fetch.c +++ b/utilities/das_fetch.c @@ -19,20 +19,12 @@ * stream their output to stdout, all by catalog path. Nobody types a * server URL. * - * The program has two halves. The small half is catalog navigation, which - * is the library's job (DasCat in das3/catalog.h, nodes in das3/node.h); only - * the command line spelling of the root policy lives here. The large half is - * turning a data source's "interface" section into command line keys, and - * turning the user's KEY=VALUE arguments back into HTTP GET parameters via - * the source's "protocol" section. The translation rules are those of - * the das catalog schema for HttpStreamSrc nodes, so any client following - * the schema agrees on what a source definition means. - * - * Two generations of HttpStreamSrc node are in the wild (0.6 from 2018, - * 0.7 from dasflex) and they spell things differently. Layout detection - * here is structural (does a group have a "props" sub-dictionary) rather - * than by version string, and the 0.6 spellings are handled inline where - * they differ. Once the 0.6 nodes are retired those branches can go. + * Everything that understands a catalog or a data source is the library's + * job: DasCat (das3/catalog.h) resolves paths, DasSrc (das3/source.h) turns + * a source node's "interface" section into keys and a DasReq (das3/req.h) + * of KEY=VALUE settings into the HTTP GET URL the "protocol" section calls + * for. What lives here is the command line spelling of all that: the root + * policy options, the example shortcuts, and the listings. * * The body of a data response is never inspected. It may be a das2 or das3 * stream, a CDF file, CSV text, or CCSDS packets. Bytes are bytes. @@ -43,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -52,6 +45,8 @@ #include #include #include +#include +#include #include #ifdef _WIN32 @@ -85,21 +80,26 @@ #define RET_AUTH 6 #define RET_WRITE 7 -#define MAX_SETTABLES 512 -#define MAX_PARAMS 96 -#define MAX_RAW 32 -/* Identifiers in a node (property, variable and group names) follow the das3 - * rule enforced by das_assert_valid_id(): at most 63 characters. A full key - * is "section.group.prop", so size it from that rule, not from a guess. */ -#define NAME_SZ DAS_MAX_ID_BUFSZ +/* Command line bounds. Identifiers in a node follow the das3 rule enforced + * by das_assert_valid_id(), at most 63 characters, and a full key is + * "section.group.prop", so KEY_SZ comes from that rule, not from a guess. */ +#define MAX_KEYS 512 /* KEY=VALUE arguments on one command line */ +#define MAX_RAW 32 /* -p arguments */ #define KEY_SZ (8 + 2*DAS_MAX_ID_BUFSZ) -#define VAL_SZ 512 +#define NAME_SZ DAS_MAX_ID_BUFSZ #define URI_SZ 1024 #define URL_SZ 4096 #define ERR_SZ 1024 #define ROOT_ENV "DASFETCH_ROOT" #define MOUNT_ENV "DASFETCH_MOUNT" +#define MAX_MOUNTS 16 +#define MAX_KIDS 32 + +/* What to take when a catalog node offers several sources. das3 streams + * are what the rest of the das3 tools read; anything else is still + * better than nothing for a program that only copies bytes to stdout. */ +#define DEF_ACCEPT "application/vnd.das.stream" /* ************************************************************************* */ /* Help */ @@ -185,6 +185,15 @@ static void prnHelpMain(void) " options, all. The default leaves out formats.\n" " -p PARAM=VALUE Send a raw HTTP GET parameter, bypassing the interface\n" " translation. Repeatable.\n" +" -a TYPES Which source to use when a catalog node offers more\n" +" than one: a comma separated list of the media types\n" +" you can read. Order means nothing, any source that\n" +" provides one of them qualifies, and */* is anything.\n" +" Without -a, das3 streams are asked for first and\n" +" then anything, as if by:\n" +" -a '" DEF_ACCEPT "' then -a '*/*'\n" +" If several sources remain, put the name of one on\n" +" the end of the path.\n" "\n" " -f Fresh: ignore cached catalog nodes along the path and\n" " fetch them again. Use after a catalog has changed.\n" @@ -236,9 +245,9 @@ static void prnHelpNode(void) " types " PROG " can list but not load or fetch from (Das2DSDF, WebSocSrc).\n" "\n" " A SourceSet or Collection holds several descriptions of one dataset.\n" -" Only the one 'data' and 'url' would use is shown: the child named\n" -" 'flex', else the first HttpStreamSrc. -v lists them all. To use\n" -" another, add its name to the path, as in uiowa/juno/wav/survey/das2.\n" +" Only the one 'data' and 'url' would use is shown, chosen by what it\n" +" provides against the -a list. -v lists them all. To use another,\n" +" add its name to the path, as in uiowa/juno/wav/survey/das2.\n" "\n" "DATA SOURCES\n" " An HttpStreamSrc node prints a header, its examples, then one table per\n" @@ -404,7 +413,7 @@ static void prnHelp(const char* sCmd) } /* ************************************************************************* */ -/* Small string and JSON helpers */ +/* Small string helpers */ static bool strieq(const char* a, const char* b) { @@ -424,154 +433,30 @@ static void strcpyz(char* sDest, const char* sSrc, size_t uDest) sDest[uDest - 1] = '\0'; } -static bool parseBool(const char* s, bool* pOut) -{ - if(s == NULL) return false; - if(strieq(s,"true")||strieq(s,"yes")||strieq(s,"on")||strieq(s,"1")|| - strieq(s,"t")||strieq(s,"y")){ *pOut = true; return true; } - if(strieq(s,"false")||strieq(s,"no")||strieq(s,"off")||strieq(s,"0")|| - strieq(s,"f")||strieq(s,"n")){ *pOut = false; return true; } - return false; -} - -/* NULL tolerant wrappers. DasJdo_get() is a slash path lookup, so the dot - * in HTTP parameter names such as "read.time.min" is not special to it. */ -static const DasJdo* jget(const DasJdo* p, const char* sPath) +static bool isDigits(const char* s) { - return (p == NULL) ? NULL : DasJdo_get(p, sPath); + if((s == NULL)||(*s == '\0')) return false; + for(; *s; ++s) if(!isdigit((unsigned char)*s)) return false; + return true; } +/* A string element of a node document, or NULL. Only the listings read + * documents directly, everything about a source comes through DasSrc. */ static const char* jstr(const DasJdo* p, const char* sPath) { - return DasJdo_string(jget(p, sPath)); -} - -static bool jisdict(const DasJdo* p){ return (p != NULL)&&(p->type == das_json_type_dict); } -static bool jisary(const DasJdo* p){ return (p != NULL)&&(p->type == das_json_type_ary); } -static bool jtrue(const DasJdo* p){ return (p != NULL)&&(p->type == das_json_type_true); } -static bool jisbool(const DasJdo* p){ - return (p != NULL)&&((p->type == das_json_type_true)||(p->type == das_json_type_false)); -} - -/* Scalar JSON value to text. False for null, missing, dicts and arrays. - * Example settings and defaults can be strings, numbers or booleans, and the - * value sent to the server is text in any case. */ -static bool jscalar(const DasJdo* p, char* sBuf, size_t uLen) -{ - if((p == NULL)||(uLen == 0)) return false; - switch(p->type){ - case das_json_type_str: - strcpyz(sBuf, ((const das_json_str*)p->value)->string, uLen); - return true; - case das_json_type_num:{ - const das_json_num* pNum = (const das_json_num*)p->value; - size_t u = (pNum->number_size < uLen - 1) ? pNum->number_size : uLen - 1; - memcpy(sBuf, pNum->number, u); - sBuf[u] = '\0'; - return true; - } - case das_json_type_true: strcpyz(sBuf, "true", uLen); return true; - case das_json_type_false: strcpyz(sBuf, "false", uLen); return true; - default: return false; - } -} - -/* Percent encode everything but RFC 3986 unreserved characters. Servers - * decode before use, so over-encoding ':' in time strings is harmless, and - * it is what most URL encoders do anyway. */ -static void urlEncode(const char* sIn, char* sOut, size_t uOut) -{ - static const char* sHex = "0123456789ABCDEF"; - size_t u = 0; - for(const unsigned char* p = (const unsigned char*)sIn; *p && (u + 4 < uOut); ++p){ - if(isalnum(*p)||(*p == '-')||(*p == '.')||(*p == '_')||(*p == '~')){ - sOut[u++] = (char)*p; - } - else{ - sOut[u++] = '%'; - sOut[u++] = sHex[*p >> 4]; - sOut[u++] = sHex[*p & 0x0F]; - } - } - sOut[u] = '\0'; -} - -/* ************************************************************************* */ -/* The settable model: a flat list of interface properties that have a 'set' - * block, with enough context to print them and to translate values. - * - * DasHttpSrc, DasSettable and DasHttpParam are named for where they are - * expected to end up, libdas3 next to DasNode, once the 1.0 catalog schema - * settles what they read (see cf_notes/api_enlargement.md). The _print and - * _describe functions are das_fetch presentation and would stay behind. */ - -typedef enum { SEC_COORD = 0, SEC_DATA, SEC_FORMAT, SEC_OPTION, SEC_COUNT } sec_e; - -/* Section names as the 0.7 catalogs spell them. Keys are built from these, - input accepts the singular forms too, see secAlias() */ -static const char* g_sSecName[SEC_COUNT] = {"coords", "data", "formats", "options"}; - -typedef struct das_settable { - sec_e sec; - char sGroup[NAME_SZ]; /* "time", "lshell", "das"; empty for options */ - char sProp[NAME_SZ]; /* "min", "enabled", "serial" */ - char sKey[KEY_SZ]; /* "coord.time.min" */ - const DasJdo* pGroup; /* the coordinate, variable, or format dict */ - const DasJdo* pProp; /* the property dict: value, set, type, label... */ - const DasJdo* pSet; /* the set dict: param, flag, pval, enum... */ - const char* sParam; /* set/param, the HTTP GET parameter name */ - const DasJdo* pHParam; /* protocol httpParams entry for sParam */ - bool bRequired; /* set/required or httpParams/required */ - bool bHasDefault; /* value is not null */ - char sDefault[VAL_SZ]; - bool bUserSet; - char sValue[VAL_SZ]; -} DasSettable; - -typedef struct das_http_param { - char sName[NAME_SZ]; - char sVal[VAL_SZ]; -} DasHttpParam; - -typedef struct das_http_src { - DasNode* pNode; - const DasJdo* pIface; - const DasJdo* pProto; - const DasJdo* pHttpParams; - const DasJdo* pBaseUrls; - bool bLegacy; /* v0.6 layout */ - DasSettable aSet[MAX_SETTABLES]; - size_t uSet; - DasHttpParam aOut[MAX_PARAMS]; - size_t uOut; -} DasHttpSrc; - -/* Property name aliases. The 0.6 nodes spell out minimum/maximum/resolution - * and interval, 0.7 nodes abbreviate. Users may type either. */ -static const char* canonProp(const char* s) -{ - if(strieq(s, "minimum")||strieq(s, "min")) return "min"; - if(strieq(s, "maximum")||strieq(s, "max")) return "max"; - if(strieq(s, "resolution")||strieq(s, "res")) return "res"; - if(strieq(s, "interval")||strieq(s, "inter")) return "inter"; - if(strieq(s, "enable")||strieq(s, "enabled")) return "enabled"; - return s; -} - -static bool propEq(const char* a, const char* b) -{ - return strieq(canonProp(a), canonProp(b)); + return (p == NULL) ? NULL : DasJdo_string(DasJdo_get(p, sPath)); } -static bool secAlias(const char* s, sec_e* pSec) +/* Section names as typed in -s lists, singular or plural */ +static bool secAlias(const char* s, das_src_sec_e* pSec) { if(strieq(s,"coord")||strieq(s,"coords")||strieq(s,"coordinate")||strieq(s,"coordinates")){ - *pSec = SEC_COORD; return true; + *pSec = das_src_sec_coord; return true; } - if(strieq(s,"data")){ *pSec = SEC_DATA; return true; } - if(strieq(s,"format")||strieq(s,"formats")||strieq(s,"fmt")){ *pSec = SEC_FORMAT; return true; } + if(strieq(s,"data")){ *pSec = das_src_sec_data; return true; } + if(strieq(s,"format")||strieq(s,"formats")||strieq(s,"fmt")){ *pSec = das_src_sec_format; return true; } if(strieq(s,"option")||strieq(s,"options")||strieq(s,"opt")||strieq(s,"opts")){ - *pSec = SEC_OPTION; return true; + *pSec = das_src_sec_option; return true; } return false; } @@ -580,9 +465,9 @@ static bool secAlias(const char* s, sec_e* pSec) /* Display settings and terminal wrapping */ /* Which parts of a source 'node' prints: one bit per section plus examples */ -#define SHOW_EXAMPLES (1u << SEC_COUNT) -#define SHOW_ALL ((1u << (SEC_COUNT + 1)) - 1) -static unsigned g_uShow = SHOW_ALL & ~(1u << SEC_FORMAT); +#define SHOW_EXAMPLES (1u << das_src_sec_size) +#define SHOW_ALL ((1u << (das_src_sec_size + 1)) - 1) +static unsigned g_uShow = SHOW_ALL & ~(1u << das_src_sec_format); static bool g_bVerbose = false; static int g_nCols = 0; /* terminal width, 0 when stdout is not a terminal */ @@ -648,748 +533,91 @@ static void prnPara(FILE* pOut, int nIndent, const char* sText) /* A labeled header line, wrapped to the terminal */ static void prnField(FILE* pOut, const char* sLabel, const char* sText) { - if(sText == NULL) return; + if((sText == NULL)||(sText[0] == '\0')) return; fprintf(pOut, "%-10s", sLabel); fputsWrapped(sText, 10, pOut); fputc('\n', pOut); } -/* Type of a settable for validation and display. Preference order: the - * property's own type, an enum in set, a boolean default, a flag toggle, - * the protocol parameter's type, then string. */ -static const char* DasSettable_type(const DasSettable* p) -{ - const char* sType = jstr(p->pProp, "type"); - if(sType != NULL){ - if(strieq(sType, "bool")) return "boolean"; - return sType; - } - if(jisary(jget(p->pSet, "enum"))) return "enum"; - if(jisbool(jget(p->pProp, "value"))) return "boolean"; - if(jstr(p->pSet, "flag") != NULL) return "boolean"; - sType = jstr(p->pHParam, "type"); - if(sType != NULL){ - if(strieq(sType, "FlagSet")||strieq(sType, "flag_set")) return "string"; - return sType; - } - return "string"; -} - -static bool DasSettable_isBool(const DasSettable* p) -{ - return strieq(DasSettable_type(p), "boolean"); -} - -/* Units for display. 0.7 puts a units string on the property; both versions - * may carry a units dict with a "value" at the group level (0.7 under props, - * 0.6 directly). */ -static const char* DasSettable_units(const DasSettable* p) -{ - const DasJdo* pU = jget(p->pProp, "units"); - if(pU != NULL){ - if(pU->type == das_json_type_str) return DasJdo_string(pU); - if(jstr(pU, "value")) return jstr(pU, "value"); - } - if(p->sec == SEC_COORD){ - const char* s = jstr(p->pGroup, "props/units/value"); - if(s == NULL) s = jstr(p->pGroup, "units/value"); - if(s != NULL) return s; - } - if(strieq(DasSettable_type(p), "isotime")) return "UTC"; - return ""; -} +/* Fixed properties are shown dimmed under -v so the eye goes to what can be + * set. Only on a terminal; a pipe gets plain text and the Req column + * says "fixed" either way. */ +static const char* dimOn(void){ return (g_nCols > 0) ? "\033[2m" : ""; } +static const char* dimOff(void){ return (g_nCols > 0) ? "\033[0m" : ""; } -static const char* groupName(const DasSettable* p) -{ - const char* s = jstr(p->pGroup, "label"); - if(s == NULL) s = jstr(p->pGroup, "name"); - if(s == NULL) s = jstr(p->pGroup, "title"); - return s ? s : p->sGroup; -} +/* ************************************************************************* */ +/* Describing a source. The library has the facts, these functions only + * decide what goes in which column. */ -/* Description column. The property's own title or label wins. Without - * one, coordinate properties have conventional meanings worth spelling out, - * and an enable toggle is best described by the thing it enables. */ -static const char* DasSettable_desc(const DasSettable* p, char* sBuf, size_t uLen) +/* The node's spelling of a type when it gave one, else a name for what + * the library inferred */ +static const char* propTypeName(const DasSrcProp* p) { - const char* s = jstr(p->pProp, "title"); - if(s == NULL) s = jstr(p->pProp, "summary"); - if(s == NULL) s = jstr(p->pProp, "label"); - if(s == NULL) s = jstr(p->pProp, "name"); - if(s != NULL) return s; - - const char* sProp = canonProp(p->sProp); - if(strcmp(sProp, "enabled") == 0){ - if(p->sec == SEC_FORMAT){ - snprintf(sBuf, uLen, "Output as %s", groupName(p)); - } - else{ - const char* sTitle = jstr(p->pGroup, "title"); - if(sTitle == NULL) sTitle = groupName(p); - snprintf(sBuf, uLen, "Output %s", sTitle); - } - return sBuf; - } - if(p->sec == SEC_COORD){ - if(strcmp(sProp, "min") == 0){ - snprintf(sBuf, uLen, "Minimum %s value to output", groupName(p)); - return sBuf; - } - if(strcmp(sProp, "max") == 0){ - snprintf(sBuf, uLen, "Maximum %s value to output", groupName(p)); - return sBuf; - } - if(strcmp(sProp, "res") == 0){ - snprintf(sBuf, uLen, "Average over %s bins of this width", groupName(p)); - return sBuf; - } - if(strcmp(sProp, "inter") == 0){ - snprintf(sBuf, uLen, "Step between output %s values", groupName(p)); - return sBuf; - } + const char* s = DasSrcProp_typeName(p); + if((s != NULL)&&(s[0] != '\0')) return s; + switch(DasSrcProp_type(p)){ + case das_srcprop_bool: return "boolean"; + case das_srcprop_int: return "integer"; + case das_srcprop_real: return "real"; + case das_srcprop_time: return "isotime"; + case das_srcprop_enum: return "enum"; + default: return "string"; } - if(p->sec != SEC_OPTION){ - snprintf(sBuf, uLen, "%s %s", groupName(p), p->sProp); - return sBuf; - } - return ""; } -/* Allowed values for display: enum entries, a range, or a coordinate's - * valid range for min/max properties. */ -static void DasSettable_values(const DasSettable* p, char* sBuf, size_t uLen) +/* Allowed values for display: enum entries, a range, or true/false */ +static void propValues(const DasSrcProp* p, char* sBuf, size_t uLen) { sBuf[0] = '\0'; - const DasJdo* pEnum = jget(p->pSet, "enum"); - if(jisary(pEnum)){ + const DasSrcCtrl* pC = DasSrcProp_ctrl(p); + if(pC == NULL) return; + + size_t uEnum = DasSrcCtrl_enumCount(pC); + if(uEnum > 0){ size_t u = 0; - char sVal[VAL_SZ]; - for(const das_json_ary_el* pEl = DasJdo_aryFirst(pEnum); pEl; pEl = pEl->next){ - if(!jscalar(jget(pEl->value, "value"), sVal, sizeof(sVal))) continue; + for(size_t i = 0; i < uEnum; ++i){ + const char* sVal = DasSrcCtrl_enum(pC, i); if(u + strlen(sVal) + 3 >= uLen){ strcpyz(sBuf + u, "...", uLen - u); return; } u += snprintf(sBuf + u, uLen - u, "%s%s", (u > 0) ? ", " : "", sVal); } return; } - const DasJdo* pRange = jget(p->pSet, "range"); - if(!jisary(pRange)) pRange = jget(p->pHParam, "range"); - if(!jisary(pRange) && (p->sec == SEC_COORD) && - (propEq(p->sProp, "min") || propEq(p->sProp, "max"))) - pRange = jget(p->pGroup, "validRange"); - if(jisary(pRange)){ - const das_json_ary_el* pEl = DasJdo_aryFirst(pRange); - char sLo[VAL_SZ] = {'\0'}, sHi[VAL_SZ] = {'\0'}; - if(pEl) jscalar(pEl->value, sLo, sizeof(sLo)); - if(pEl && pEl->next) jscalar(pEl->next->value, sHi, sizeof(sHi)); + const char* sLo = ""; + const char* sHi = ""; + if(DasSrcCtrl_range(pC, &sLo, &sHi)){ snprintf(sBuf, uLen, "%s to %s", sLo, sHi); return; } - if(DasSettable_isBool(p)){ - strcpyz(sBuf, "true, false", uLen); - } -} - -/* ************************************************************************* */ -/* Flattening the interface */ - -static void DasHttpSrc_addProp( - DasHttpSrc* pS, sec_e sec, const char* sGroup, const DasJdo* pGroup, - const char* sProp, const DasJdo* pProp -){ - if(!jisdict(pProp)) return; - const DasJdo* pSet = jget(pProp, "set"); - if(!jisdict(pSet)) return; /* documentation only, not settable */ - - if(pS->uSet >= MAX_SETTABLES){ - daslog_warn_v("Source lists more than %d settable properties, ignoring %s", - MAX_SETTABLES, sProp); - return; - } - DasSettable* p = pS->aSet + pS->uSet; - memset(p, 0, sizeof(DasSettable)); - pS->uSet += 1; - - p->sec = sec; - strcpyz(p->sGroup, sGroup, NAME_SZ); - strcpyz(p->sProp, sProp, NAME_SZ); - if(sec == SEC_OPTION) - snprintf(p->sKey, KEY_SZ, "%s.%s", g_sSecName[sec], sProp); - else - snprintf(p->sKey, KEY_SZ, "%s.%s.%s", g_sSecName[sec], sGroup, sProp); - - p->pGroup = pGroup; - p->pProp = pProp; - p->pSet = pSet; - p->sParam = jstr(pSet, "param"); - p->pHParam = p->sParam ? jget(pS->pHttpParams, p->sParam) : NULL; - p->bRequired = jtrue(jget(pSet, "required")) || jtrue(jget(p->pHParam, "required")); - p->bHasDefault = jscalar(jget(pProp, "value"), p->sDefault, VAL_SZ); -} - -/* A group is a coordinate, a data variable, a format, or the whole options - * block. 0.7 groups keep properties under "props", 0.6 groups hold them - * directly beside label/title/units. Anything without a 'set' is skipped - * either way, so iterating the group itself is safe for 0.6. */ -static void DasHttpSrc_addGroup(DasHttpSrc* pS, sec_e sec, const char* sGroup, const DasJdo* pGroup) -{ - if(!jisdict(pGroup)) return; - const DasJdo* pProps = jget(pGroup, "props"); - const DasJdo* pIter = jisdict(pProps) ? pProps : pGroup; - for(const das_json_dict_el* pEl = DasJdo_dictFirst(pIter); pEl; pEl = pEl->next) - DasHttpSrc_addProp(pS, sec, sGroup, pGroup, pEl->name->string, pEl->value); -} - -static void DasHttpSrc_addSection(DasHttpSrc* pS, sec_e sec, const DasJdo* pSec) -{ - if(!jisdict(pSec)) return; - for(const das_json_dict_el* pEl = DasJdo_dictFirst(pSec); pEl; pEl = pEl->next) - DasHttpSrc_addGroup(pS, sec, pEl->name->string, pEl->value); -} - -static bool DasHttpSrc_init(DasHttpSrc* pS, DasNode* pNode, char* sErr, size_t uErr) -{ - memset(pS, 0, sizeof(DasHttpSrc)); - pS->pNode = pNode; - const DasJdo* pDom = DasNode_getJdo(pNode, NULL); - pS->pIface = jget(pDom, "interface"); - pS->pProto = jget(pDom, "protocol"); - if(!jisdict(pS->pProto)){ - snprintf(sErr, uErr, "Source node %s has no 'protocol' section", DasNode_srcUrl(pNode)); - return false; - } - pS->pHttpParams = jget(pS->pProto, "httpParams"); - if(pS->pHttpParams == NULL) pS->pHttpParams = jget(pS->pProto, "http_params"); - pS->pBaseUrls = jget(pS->pProto, "baseUrls"); - if(pS->pBaseUrls == NULL) pS->pBaseUrls = jget(pS->pProto, "base_urls"); - if(!jisary(pS->pBaseUrls)){ - snprintf(sErr, uErr, "Source node %s has no base URL list", DasNode_srcUrl(pNode)); - return false; - } - pS->bLegacy = (jget(pS->pProto, "http_params") != NULL) || - (jget(pS->pIface, "coordinates") != NULL); - - const DasJdo* pSec = jget(pS->pIface, "coords"); - if(pSec == NULL) pSec = jget(pS->pIface, "coordinates"); - DasHttpSrc_addSection(pS, SEC_COORD, pSec); - DasHttpSrc_addSection(pS, SEC_DATA, jget(pS->pIface, "data")); - DasHttpSrc_addSection(pS, SEC_FORMAT, jget(pS->pIface, "formats")); - DasHttpSrc_addGroup(pS, SEC_OPTION, "", jget(pS->pIface, "options")); - return true; -} - -/* ************************************************************************* */ -/* Key lookup */ - -/* Split "a.b.c" into parts, returns the count or 0 if malformed */ -static int splitKey(const char* sKey, char aParts[3][NAME_SZ]) -{ - int n = 0; - const char* pBeg = sKey; - while(true){ - const char* pEnd = strchr(pBeg, '.'); - size_t uLen = pEnd ? (size_t)(pEnd - pBeg) : strlen(pBeg); - if((uLen == 0)||(uLen >= NAME_SZ)||(n == 3)) return 0; - memcpy(aParts[n], pBeg, uLen); - aParts[n][uLen] = '\0'; - ++n; - if(pEnd == NULL) break; - pBeg = pEnd + 1; - } - return n; -} - -/* Find settables matching a dotted key. Returns the match count and fills - * ppOut with up to nMax of them. More than one match is an error the caller - * reports with the candidate list; zero means no such key. */ -static int DasHttpSrc_match(DasHttpSrc* pS, const char* sKey, DasSettable** ppOut, int nMax) -{ - char aParts[3][NAME_SZ]; - int nParts = splitKey(sKey, aParts); - if(nParts == 0) return 0; - - int n = 0; - sec_e sec; - DasSettable* p = NULL; - -#define _ADD_MATCH if(n < nMax) ppOut[n] = p; ++n; - - if(nParts == 3){ - if(!secAlias(aParts[0], &sec)) return 0; - for(size_t u = 0; u < pS->uSet; ++u){ - p = pS->aSet + u; - if((p->sec == sec) && strieq(p->sGroup, aParts[1]) && propEq(p->sProp, aParts[2])){ - _ADD_MATCH - } - } - return n; - } - - if(nParts == 2){ - /* "option.X" is a section qualified option, "SEC.GROUP" toggles a - variable, anything else is GROUP.PROP in any section */ - if(secAlias(aParts[0], &sec)){ - for(size_t u = 0; u < pS->uSet; ++u){ - p = pS->aSet + u; - if(p->sec != sec) continue; - if((sec == SEC_OPTION) && propEq(p->sProp, aParts[1])){ _ADD_MATCH } - else if((sec != SEC_OPTION) && strieq(p->sGroup, aParts[1]) && - propEq(p->sProp, "enabled")){ _ADD_MATCH } - } - if(n > 0) return n; - } - for(size_t u = 0; u < pS->uSet; ++u){ - p = pS->aSet + u; - if(strieq(p->sGroup, aParts[0]) && propEq(p->sProp, aParts[1])){ _ADD_MATCH } - } - return n; - } - - /* One part: an option, or a variable/format toggle */ - for(size_t u = 0; u < pS->uSet; ++u){ - p = pS->aSet + u; - if((p->sec == SEC_OPTION) && propEq(p->sProp, aParts[0])){ _ADD_MATCH } - else if((p->sec != SEC_OPTION) && strieq(p->sGroup, aParts[0]) && - propEq(p->sProp, "enabled")){ _ADD_MATCH } - } - if(n > 0) return n; - - /* Last resort, a bare property name. Unique in practice for things - like 'serial', ambiguous for 'fracSecs' shared by das and csv, and - the ambiguity message then lists the full keys. */ - for(size_t u = 0; u < pS->uSet; ++u){ - p = pS->aSet + u; - if((p->sec != SEC_OPTION) && propEq(p->sProp, aParts[0])){ _ADD_MATCH } - } -#undef _ADD_MATCH - return n; -} - -/* If sKey (with or without a coord. prefix) names a coordinate group that - * has settable properties, return the group name as spelled by the node */ -static const char* DasHttpSrc_coordGroup(DasHttpSrc* pS, const char* sKey) -{ - char aParts[3][NAME_SZ]; - int nParts = splitKey(sKey, aParts); - const char* sGroup = NULL; - sec_e sec; - if(nParts == 1) sGroup = aParts[0]; - else if((nParts == 2) && secAlias(aParts[0], &sec) && (sec == SEC_COORD)) sGroup = aParts[1]; - else return NULL; - - for(size_t u = 0; u < pS->uSet; ++u){ - if((pS->aSet[u].sec == SEC_COORD) && strieq(pS->aSet[u].sGroup, sGroup)) - return pS->aSet[u].sGroup; - } - return NULL; -} - -static bool DasHttpSrc_hasProp(const DasHttpSrc* pS, sec_e sec, const char* sGroup, const char* sProp) -{ - for(size_t u = 0; u < pS->uSet; ++u){ - const DasSettable* p = pS->aSet + u; - if((p->sec == sec) && strieq(p->sGroup, sGroup) && propEq(p->sProp, sProp)) - return true; - } - return false; -} - -static DasSettable* DasHttpSrc_find(DasHttpSrc* pS, sec_e sec, const char* sGroup, const char* sProp) -{ - for(size_t u = 0; u < pS->uSet; ++u){ - DasSettable* p = pS->aSet + u; - if((p->sec == sec) && strieq(p->sGroup, sGroup) && propEq(p->sProp, sProp)) - return p; - } - return NULL; -} - -/* Validate and store one user value. */ -static bool DasSettable_set(DasSettable* p, const char* sVal, char* sErr, size_t uErr) -{ - const char* sType = DasSettable_type(p); - - if(strieq(sType, "boolean")){ - bool b; - if(!parseBool(sVal, &b)){ - snprintf(sErr, uErr, "'%s' is not a boolean value for %s, use true or false", - sVal, p->sKey); - return false; - } - strcpyz(p->sValue, b ? "true" : "false", VAL_SZ); - p->bUserSet = true; - return true; - } - - const DasJdo* pEnum = jget(p->pSet, "enum"); - if(jisary(pEnum)){ - char sItem[VAL_SZ]; - for(const das_json_ary_el* pEl = DasJdo_aryFirst(pEnum); pEl; pEl = pEl->next){ - if(jscalar(jget(pEl->value, "value"), sItem, sizeof(sItem)) && strieq(sItem, sVal)){ - strcpyz(p->sValue, sItem, VAL_SZ); /* node's spelling */ - p->bUserSet = true; - return true; - } - } - char sVals[ERR_SZ/2]; - DasSettable_values(p, sVals, sizeof(sVals)); - snprintf(sErr, uErr, "'%s' is not an allowed value for %s, choose one of: %s", - sVal, p->sKey, sVals); - return false; - } - - strcpyz(p->sValue, sVal, VAL_SZ); - p->bUserSet = true; - return true; -} - -/* Apply one KEY=VALUE from the command line, including the shortcuts. */ -static bool DasHttpSrc_set(DasHttpSrc* pS, const char* sKey, const char* sVal, char* sErr, size_t uErr) -{ - /* Coordinate range shortcut: GROUP=MIN,MAX[,STEP] */ - const char* sGroup = DasHttpSrc_coordGroup(pS, sKey); - if(sGroup != NULL){ - const char* aProps[3] = {"min", "max", NULL}; - aProps[2] = DasHttpSrc_hasProp(pS, SEC_COORD, sGroup, "res") ? "res" : "inter"; - - char sCopy[VAL_SZ]; - strcpyz(sCopy, sVal, VAL_SZ); - char* pTok = sCopy; - for(int i = 0; i < 4; ++i){ - char* pComma = strchr(pTok, ','); - if(pComma) *pComma = '\0'; - if(i == 3){ - snprintf(sErr, uErr, "Too many values for coordinate %s, at most MIN,MAX,STEP", sGroup); - return false; - } - if(pTok[0] != '\0'){ - DasSettable* p = DasHttpSrc_find(pS, SEC_COORD, sGroup, aProps[i]); - if(p == NULL){ - snprintf(sErr, uErr, "Coordinate %s has no settable '%s' property", sGroup, aProps[i]); - return false; - } - if(!DasSettable_set(p, pTok, sErr, uErr)) return false; - } - if(pComma == NULL) break; - pTok = pComma + 1; - } - return true; - } - - DasSettable* aMatch[8]; - int n = DasHttpSrc_match(pS, sKey, aMatch, 8); - if(n == 0){ - snprintf(sErr, uErr, "No query key named '%s' for this source, run '" PROG - " node PATH' to list them", sKey); - return false; - } - if(n > 1){ - size_t u = snprintf(sErr, uErr, "Key '%s' is ambiguous, use one of:", sKey); - for(int i = 0; (i < n) && (i < 8) && (u < uErr); ++i) - u += snprintf(sErr + u, uErr - u, " %s", aMatch[i]->sKey); - return false; - } - return DasSettable_set(aMatch[0], sVal, sErr, uErr); -} - -/* ************************************************************************* */ -/* Translation to protocol parameters */ - -static DasHttpParam* DasHttpSrc_outFind(DasHttpSrc* pS, const char* sName) -{ - for(size_t u = 0; u < pS->uOut; ++u) - if(strcmp(pS->aOut[u].sName, sName) == 0) return pS->aOut + u; - return NULL; -} - -static bool DasHttpSrc_outSet(DasHttpSrc* pS, const char* sName, const char* sVal, char* sErr, size_t uErr) -{ - DasHttpParam* p = DasHttpSrc_outFind(pS, sName); - if(p == NULL){ - if(pS->uOut >= MAX_PARAMS){ - snprintf(sErr, uErr, "More than %d HTTP parameters in one query", MAX_PARAMS); - return false; - } - p = pS->aOut + pS->uOut; - pS->uOut += 1; - strcpyz(p->sName, sName, NAME_SZ); - p->sVal[0] = '\0'; - } - strcpyz(p->sVal, sVal, VAL_SZ); - return true; -} - -/* Flag set parameters accumulate: several interface toggles map to one - * parameter whose value is the flags joined by a separator */ -static bool DasHttpSrc_outAddFlag( - DasHttpSrc* pS, const char* sName, const char* sFlagVal, const char* sSep, - char* sErr, size_t uErr -){ - DasHttpParam* p = DasHttpSrc_outFind(pS, sName); - if((p == NULL)||(p->sVal[0] == '\0')) - return DasHttpSrc_outSet(pS, sName, sFlagVal, sErr, uErr); - if(strlen(p->sVal) + strlen(sSep) + strlen(sFlagVal) >= VAL_SZ){ - snprintf(sErr, uErr, "Value for HTTP parameter %s is too long", sName); - return false; - } - strcat(p->sVal, sSep); - strcat(p->sVal, sFlagVal); - return true; -} - -static bool DasHttpSrc_translate(DasHttpSrc* pS, DasSettable* p, const char* sVal, char* sErr, size_t uErr) -{ - if(p->sParam == NULL){ - snprintf(sErr, uErr, "Key %s has no 'param' in its set block, the node is broken", p->sKey); - return false; - } - - char sFlag[NAME_SZ] = {'\0'}; - char sPVal[VAL_SZ] = {'\0'}; - bool bHavePVal = false; - strcpyz(sFlag, jstr(p->pSet, "flag"), NAME_SZ); - bHavePVal = jscalar(jget(p->pSet, "pval"), sPVal, VAL_SZ); - - const DasJdo* pEnum = jget(p->pSet, "enum"); - if(jisary(pEnum)){ - /* DasSettable_set already checked membership, now pick up per-item - overrides of the protocol value or flag */ - char sItem[VAL_SZ]; - for(const das_json_ary_el* pEl = DasJdo_aryFirst(pEnum); pEl; pEl = pEl->next){ - if(!jscalar(jget(pEl->value, "value"), sItem, sizeof(sItem))) continue; - if(!strieq(sItem, sVal)) continue; - if(jscalar(jget(pEl->value, "pval"), sPVal, VAL_SZ)) bHavePVal = true; - if(jstr(pEl->value, "flag")) strcpyz(sFlag, jstr(pEl->value, "flag"), NAME_SZ); - break; - } - } - else if(DasSettable_isBool(p)){ - bool bUser = false; - parseBool(sVal, &bUser); - const DasJdo* pTrig = jget(p->pSet, "value"); - if(jisbool(pTrig)){ - /* The set block fires for one state only, usually true. The other - state is the default and has nothing to send. */ - if(bUser != jtrue(pTrig)) return true; - } - if(!bHavePVal && (sFlag[0] == '\0')){ - strcpyz(sPVal, bUser ? "true" : "false", VAL_SZ); - bHavePVal = true; - } - } - - if(sFlag[0] != '\0'){ - char sPath[NAME_SZ + 8]; - snprintf(sPath, sizeof(sPath), "flags/%s", sFlag); - const DasJdo* pF = jget(p->pHParam, sPath); - if(pF == NULL){ - snprintf(sErr, uErr, "Flag '%s' for key %s is not defined under protocol " - "parameter %s, the node is broken", sFlag, p->sKey, p->sParam); - return false; - } - char sFVal[VAL_SZ]; - if(!jscalar(jget(pF, "value"), sFVal, VAL_SZ)) strcpyz(sFVal, sFlag, VAL_SZ); - const char* sPrefix = jstr(pF, "prefix"); - if(sPrefix != NULL){ - char sTmp[VAL_SZ]; - snprintf(sTmp, VAL_SZ, "%s%s", sPrefix, sFVal); - strcpyz(sFVal, sTmp, VAL_SZ); - } - const char* sSep = jstr(p->pHParam, "flagSep"); - if(sSep == NULL) sSep = jstr(p->pHParam, "flag_sep"); - if(sSep == NULL) sSep = " "; - return DasHttpSrc_outAddFlag(pS, p->sParam, sFVal, sSep, sErr, uErr); - } - - return DasHttpSrc_outSet(pS, p->sParam, bHavePVal ? sPVal : sVal, sErr, uErr); -} - -/* Turn everything the user set (plus required defaults) into parameters. - * Adds to pS->aOut, overriding same named entries, so a 0.6 example's raw - * protocol parameters can be loaded first and still lose to the user's - * keys. */ -static bool DasHttpSrc_build(DasHttpSrc* pS, char* sErr, size_t uErr) -{ - for(size_t u = 0; u < pS->uSet; ++u){ - DasSettable* p = pS->aSet + u; - const char* sVal = NULL; - - if(p->bUserSet) sVal = p->sValue; - else if(p->bRequired && p->bHasDefault) sVal = p->sDefault; - else continue; - - /* A required default fills a hole, it never replaces a value that an - example or -p already supplied */ - if(!p->bUserSet && p->sParam && DasHttpSrc_outFind(pS, p->sParam)) continue; - - /* A value equal to the default carries no information unless the - server insists on hearing it */ - if(p->bUserSet && !p->bRequired && p->bHasDefault){ - bool bSame = false; - if(DasSettable_isBool(p)){ - bool bA = false, bB = false; - bSame = parseBool(sVal, &bA) && parseBool(p->sDefault, &bB) && (bA == bB); - } - else{ - bSame = strieq(sVal, p->sDefault); - } - if(bSame) continue; - } - if(!DasHttpSrc_translate(pS, p, sVal, sErr, uErr)) return false; - } - return true; -} - -/* Every protocol parameter marked required must be present. Servers - * enforce this, and their reply is a 400 whose body we never see, so say it - * here in terms of the interface key the user can set. */ -static bool DasHttpSrc_checkRequired(DasHttpSrc* pS, char* sErr, size_t uErr) -{ - if(!jisdict(pS->pHttpParams)) return true; - for(const das_json_dict_el* pEl = DasJdo_dictFirst(pS->pHttpParams); pEl; pEl = pEl->next){ - if(!jtrue(jget(pEl->value, "required"))) continue; - const char* sName = pEl->name->string; - if(DasHttpSrc_outFind(pS, sName) != NULL) continue; - - size_t u = snprintf(sErr, uErr, "Required parameter missing, set it with"); - int nKeys = 0; - for(size_t i = 0; (i < pS->uSet) && (u < uErr); ++i){ - if(pS->aSet[i].sParam && (strcmp(pS->aSet[i].sParam, sName) == 0)){ - u += snprintf(sErr + u, uErr - u, "%s %s=VALUE", nKeys ? " or" : "", pS->aSet[i].sKey); - ++nKeys; - } - } - if(nKeys == 0) - snprintf(sErr + u, uErr - u, " -p %s=VALUE (no interface key sets it)", sName); - else - snprintf(sErr + u, uErr - u, " (HTTP parameter %s)", sName); - return false; - } - return true; -} - -static size_t DasHttpSrc_baseCount(const DasHttpSrc* pS) -{ - return jisary(pS->pBaseUrls) ? ((const das_json_ary*)pS->pBaseUrls->value)->length : 0; -} - -static const char* DasHttpSrc_baseUrl(const DasHttpSrc* pS, size_t uIdx) -{ - const das_json_ary_el* pEl = DasJdo_aryFirst(pS->pBaseUrls); - for(size_t u = 0; (u < uIdx) && pEl; ++u) pEl = pEl->next; - return pEl ? DasJdo_string(pEl->value) : NULL; -} - -/* Base URL plus the parameters. 0.6 base URLs already carry a query string - * (server=dataset&dataset=...), 0.7 ones do not, hence the joiner check. */ -static bool DasHttpSrc_url(const DasHttpSrc* pS, size_t uBase, char* sUrl, size_t uLen) -{ - const char* sBase = DasHttpSrc_baseUrl(pS, uBase); - if(sBase == NULL) return false; - size_t u = snprintf(sUrl, uLen, "%s", sBase); - const char* sJoin = strchr(sBase, '?') ? "&" : "?"; - char sEnc[VAL_SZ * 3]; - for(size_t i = 0; (i < pS->uOut) && (u < uLen); ++i){ - urlEncode(pS->aOut[i].sVal, sEnc, sizeof(sEnc)); - u += snprintf(sUrl + u, uLen - u, "%s%s=%s", sJoin, pS->aOut[i].sName, sEnc); - sJoin = "&"; - } - return u < uLen; -} - -/* ************************************************************************* */ -/* Examples */ - -/* 0.7 example settings are keyed by interface path, "coords/time/props/min". - * Convert to the dotted key form: drop "props", singular section name. */ -static bool exPathToKey(const char* sPath, char* sKey, size_t uLen) -{ - char sCopy[KEY_SZ * 2]; - strcpyz(sCopy, sPath, sizeof(sCopy)); - size_t u = 0; - int nTok = 0; - sec_e sec; - for(char* pTok = strtok(sCopy, "/"); pTok; pTok = strtok(NULL, "/")){ - if(strcmp(pTok, "props") == 0) continue; - const char* sOut = pTok; - if((nTok == 0) && secAlias(pTok, &sec)) sOut = g_sSecName[sec]; - if(u + strlen(sOut) + 2 >= uLen) return false; - u += snprintf(sKey + u, uLen - u, "%s%s", nTok ? "." : "", sOut); - ++nTok; - } - return nTok > 0; -} - -static size_t DasHttpSrc_exampleCount(const DasHttpSrc* pS) -{ - if(pS->bLegacy){ - const DasJdo* pEx = jget(pS->pProto, "examples"); - return jisdict(pEx) ? ((const das_json_dict*)pEx->value)->length : 0; - } - const DasJdo* pEx = jget(pS->pIface, "examples"); - return jisary(pEx) ? ((const das_json_ary*)pEx->value)->length : 0; + if(DasSrcProp_type(p) == das_srcprop_bool) strcpyz(sBuf, "true, false", uLen); } -/* The example's own JSON object, or NULL if there is no such example */ -static const DasJdo* DasHttpSrc_example(const DasHttpSrc* pS, size_t uIdx, const char** psName) +/* Print the arguments that reproduce an example, indented under its row. + * The library already has them in interface terms, whatever the node + * layout, so this is a requisition read back as KEY=VALUE pairs. */ +static void printExampleArgs(const DasSrc* pSrc, size_t uIdx, int nCol, FILE* pOut) { - if(psName) *psName = NULL; - if(pS->bLegacy){ - const das_json_dict_el* pEl = DasJdo_dictFirst(jget(pS->pProto, "examples")); - for(size_t u = 0; (u < uIdx) && pEl; ++u) pEl = pEl->next; - if(pEl == NULL) return NULL; - if(psName) *psName = pEl->name->string; - return pEl->value; - } - const das_json_ary_el* pEl = DasJdo_aryFirst(jget(pS->pIface, "examples")); - for(size_t u = 0; (u < uIdx) && pEl; ++u) pEl = pEl->next; - return pEl ? pEl->value : NULL; -} - -static const char* DasHttpSrc_exampleLabel(const DasHttpSrc* pS, size_t uIdx) -{ - const char* sName = NULL; - const DasJdo* pEx = DasHttpSrc_example(pS, uIdx, &sName); - if(pEx == NULL) return ""; - const char* sLabel = jstr(pEx, "label"); - if(sLabel == NULL) sLabel = jstr(pEx, "title"); - if(sLabel == NULL) sLabel = sName; - return sLabel ? sLabel : ""; -} - -/* Print the arguments that reproduce an example, indented under its row */ -static void DasHttpSrc_printExampleArgs( - const DasHttpSrc* pS, size_t uIdx, int nCol, FILE* pOut -){ - char sKey[KEY_SZ], sVal[VAL_SZ]; + DasReq* pReq = DasSrc_newReq(pSrc, (int)uIdx); + if(pReq == NULL) return; char sArgs[4096] = {'\0'}; size_t u = 0; - const DasJdo* pEx = DasHttpSrc_example(pS, uIdx, NULL); - if(pEx == NULL) return; + for(size_t i = 0; (i < DasReq_count(pReq)) && (u < sizeof(sArgs)); ++i) + u += snprintf(sArgs + u, sizeof(sArgs) - u, "%s=%s ", DasReq_key(pReq, i), DasReq_val(pReq, i)); + for(size_t i = 0; (i < DasReq_rawCount(pReq)) && (u < sizeof(sArgs)); ++i){ + const char* sVal = NULL; + const char* sName = DasReq_raw(pReq, i, &sVal); + u += snprintf(sArgs + u, sizeof(sArgs) - u, "-p %s=%s ", sName, sVal); + } + del_DasReq(pReq); for(int i = 0; i < nCol; ++i) fputc(' ', pOut); - if(pS->bLegacy){ - u += snprintf(sArgs + u, sizeof(sArgs) - u, "(protocol parameters) "); - const DasJdo* pParams = jget(pEx, "http_params"); - for(const das_json_dict_el* pP = DasJdo_dictFirst(pParams); pP && (u < sizeof(sArgs)); pP = pP->next){ - if(jscalar(pP->value, sVal, sizeof(sVal))) - u += snprintf(sArgs + u, sizeof(sArgs) - u, "%s=%s ", pP->name->string, sVal); - } - } - else{ - const DasJdo* pSettings = jget(pEx, "settings"); - for(const das_json_dict_el* pP = DasJdo_dictFirst(pSettings); pP && (u < sizeof(sArgs)); pP = pP->next){ - if(exPathToKey(pP->name->string, sKey, sizeof(sKey)) && jscalar(pP->value, sVal, sizeof(sVal))) - u += snprintf(sArgs + u, sizeof(sArgs) - u, "%s=%s ", sKey, sVal); - } - } fputsWrapped(sArgs, nCol, pOut); fputs("\n", pOut); } /* The examples table: number and what it provides, arguments only with -v */ -static void DasHttpSrc_printExamples(const DasHttpSrc* pS, FILE* pOut) +static void printExamples(const DasSrc* pSrc, FILE* pOut) { - size_t uEx = DasHttpSrc_exampleCount(pS); + size_t uEx = DasSrc_exampleCount(pSrc); if(uEx == 0) return; size_t wKey = 3; char sNum[16]; @@ -1403,71 +631,24 @@ static void DasHttpSrc_printExamples(const DasHttpSrc* pS, FILE* pOut) int nCol = 2 + (int)wKey + 2; for(size_t u = 0; u < uEx; ++u){ fprintf(pOut, " %-*zu ", (int)wKey, u); - fputsWrapped(DasHttpSrc_exampleLabel(pS, u), nCol, pOut); + fputsWrapped(DasSrc_exampleLabel(pSrc, u), nCol, pOut); fputc('\n', pOut); - if(g_bVerbose) DasHttpSrc_printExampleArgs(pS, u, nCol, pOut); - } -} - -/* Load example uIdx. 0.7 examples set interface keys and go through the - * normal translation. 0.6 examples are protocol parameters and are applied - * raw after translation, see main(). */ -static bool DasHttpSrc_applyExample( - DasHttpSrc* pS, size_t uIdx, DasHttpParam* aRaw, size_t* puRaw, size_t uMaxRaw, - char* sErr, size_t uErr -){ - if(uIdx >= DasHttpSrc_exampleCount(pS)){ - snprintf(sErr, uErr, "Source has %zu example(s), there is no example %zu", - DasHttpSrc_exampleCount(pS), uIdx); - return false; - } - char sKey[KEY_SZ], sVal[VAL_SZ]; - if(pS->bLegacy){ - const das_json_dict_el* pEl = DasJdo_dictFirst(jget(pS->pProto, "examples")); - for(size_t u = 0; u < uIdx; ++u) pEl = pEl->next; - const DasJdo* pParams = jget(pEl->value, "http_params"); - for(const das_json_dict_el* pP = DasJdo_dictFirst(pParams); pP; pP = pP->next){ - if(!jscalar(pP->value, sVal, sizeof(sVal))) continue; - if(*puRaw >= uMaxRaw){ - snprintf(sErr, uErr, "Example %zu has more than %zu parameters", uIdx, uMaxRaw); - return false; - } - strcpyz(aRaw[*puRaw].sName, pP->name->string, NAME_SZ); - strcpyz(aRaw[*puRaw].sVal, sVal, VAL_SZ); - *puRaw += 1; - } - return true; + if(g_bVerbose) printExampleArgs(pSrc, u, nCol, pOut); } - const das_json_ary_el* pEl = DasJdo_aryFirst(jget(pS->pIface, "examples")); - for(size_t u = 0; u < uIdx; ++u) pEl = pEl->next; - const DasJdo* pSettings = jget(pEl->value, "settings"); - for(const das_json_dict_el* pP = DasJdo_dictFirst(pSettings); pP; pP = pP->next){ - if(!exPathToKey(pP->name->string, sKey, sizeof(sKey))){ - snprintf(sErr, uErr, "Can't interpret example setting path '%s'", pP->name->string); - return false; - } - if(!jscalar(pP->value, sVal, sizeof(sVal))) continue; - if(!DasHttpSrc_set(pS, sKey, sVal, sErr, uErr)) return false; - } - return true; } -/* ************************************************************************* */ -/* Stream version and format selection */ - -/* ************************************************************************* */ -/* Describing a source */ - -static void prnAuth(const DasHttpSrc* pS, FILE* pOut) +/* The realm and methods are not modeled by DasSrc, they come from the + * protocol section as the node wrote it. Old nodes say "authentication". */ +static void prnAuth(const DasSrc* pSrc, FILE* pOut) { - const DasJdo* pAuth = jget(pS->pProto, "authorization"); - if(pAuth == NULL) pAuth = jget(pS->pProto, "authentication"); + const DasJdo* pAuth = DasSrc_jdo(pSrc, "protocol/authorization"); + if(pAuth == NULL) pAuth = DasSrc_jdo(pSrc, "protocol/authentication"); if(pAuth == NULL){ fputs("Auth: unknown\n", pOut); return; } - if(!jtrue(jget(pAuth, "required"))){ fputs("Auth: not required\n", pOut); return; } + if(!DasSrc_needsAuth(pSrc)){ fputs("Auth: not required\n", pOut); return; } fputs("Auth: REQUIRED", pOut); if(jstr(pAuth, "realm")) fprintf(pOut, ", realm \"%s\"", jstr(pAuth, "realm")); - const DasJdo* pMeth = jget(pAuth, "methods"); - if(jisary(pMeth)){ + const DasJdo* pMeth = DasJdo_get(pAuth, "methods"); + if((pMeth != NULL)&&(pMeth->type == das_json_type_ary)){ fputs(" via", pOut); for(const das_json_ary_el* pEl = DasJdo_aryFirst(pMeth); pEl; pEl = pEl->next) fprintf(pOut, " %s", DasJdo_string(pEl->value) ? DasJdo_string(pEl->value) : "?"); @@ -1476,89 +657,67 @@ static void prnAuth(const DasHttpSrc* pS, FILE* pOut) fputs("\n", pOut); } -static void prnFormats(const DasHttpSrc* pS, FILE* pOut) +/* The format groups by name, or for a source without any the one MIME + * type it produces */ +static void prnFormats(const DasSrc* pSrc, FILE* pOut) { - const DasJdo* pFmts = jget(pS->pIface, "formats"); - if(jisdict(pFmts)){ + size_t uGroups = DasSrc_groupCount(pSrc, das_src_sec_format); + if(uGroups > 0){ fputs("Formats: ", pOut); - for(const das_json_dict_el* pEl = DasJdo_dictFirst(pFmts); pEl; pEl = pEl->next){ - fprintf(pOut, " %s", pEl->name->string); - if(jtrue(jget(pEl->value, "props/enabled/value"))) fputs(" (default)", pOut); + for(size_t g = 0; g < uGroups; ++g){ + const DasSrcGroup* pG = DasSrc_group(pSrc, das_src_sec_format, g); + fprintf(pOut, " %s", DasSrcGroup_name(pG)); + const DasSrcProp* pOn = DasSrcGroup_enable(pG); + if((pOn != NULL) && strieq(DasSrcProp_value(pOn), "true")) fputs(" (default)", pOut); } fputs("\n", pOut); return; } - const char* sMime = jstr(pS->pProto, "format/default/mime"); - if(sMime == NULL) sMime = jstr(pS->pIface, "format/default/mime"); - if(sMime == NULL){ - const DasJdo* pDom = DasNode_getJdo(pS->pNode, NULL); - sMime = jstr(pDom, "format/default/mime"); - } - if(sMime) fprintf(pOut, "Formats: %s\n", sMime); + const char* sMime = DasSrc_mime(pSrc, NULL); + if(sMime[0] != '\0') fprintf(pOut, "Formats: %s\n", sMime); } -static void DasHttpSrc_describe(const DasHttpSrc* pS, FILE* pOut, const char* sLabel, const char* sUri) +/* One table per section. Keys are shown without their section prefix + * since the heading carries it. Column widths are per section so the long + * enum lists of the formats section do not push everything else over. + * Fixed properties, the ones with no control, only appear under -v. */ +static void printKeyTables(const DasSrc* pSrc, FILE* pOut) { - const DasNode* pNode = pS->pNode; - const char* sVer = DasNode_version(pNode); - fprintf(pOut, "HttpStreamSrc%s%s: %s\n", sVer ? " " : "", sVer ? sVer : "", - DasNode_label(pNode) ? DasNode_label(pNode) : ""); - prnField(pOut, "Title:", DasNode_title(pNode)); - prnField(pOut, "Summary:", DasNode_summary(pNode)); - prnField(pOut, sLabel, sUri); - if(g_bVerbose){ - prnField(pOut, "From:", DasNode_srcUrl(pNode)); - const char* sBase = DasHttpSrc_baseUrl(pS, 0); - if(sBase){ - fprintf(pOut, "Server: %s", sBase); - if(DasHttpSrc_baseCount(pS) > 1) fprintf(pOut, " (+%zu alternate)", DasHttpSrc_baseCount(pS) - 1); - fputs("\n", pOut); - } - } - prnAuth(pS, pOut); - prnFormats(pS, pOut); - - if(g_uShow & SHOW_EXAMPLES) DasHttpSrc_printExamples(pS, pOut); - - if(pS->uSet == 0){ - fputs("\nThis source takes no query parameters.\n", pOut); - return; - } - - if(g_bVerbose) fputs("\nQuery keys, use as KEY=VALUE with 'data' or 'url':\n", pOut); - - /* One table per section, keys shown without their section prefix since - the heading carries it. Column widths are per section so the long - enum lists of the formats section do not push everything else over. */ enum { C_KEY, C_TYPE, C_UNITS, C_DEF, C_REQ, C_VALS, C_DESC, C_COUNT }; const char* aHdr[C_COUNT] = {"Key", "Type", "Units", "Default", "Req", "Values", "Description"}; char sVals[80]; - char sDesc[256]; + char sTitle[64]; - for(int iSec = 0; iSec < SEC_COUNT; ++iSec){ - sec_e sec = (sec_e)iSec; - size_t uInSec = 0; - for(size_t u = 0; u < pS->uSet; ++u) if(pS->aSet[u].sec == sec) ++uInSec; - if(uInSec == 0) continue; + for(int iSec = 0; iSec < das_src_sec_size; ++iSec){ + das_src_sec_e sec = (das_src_sec_e)iSec; if(!(g_uShow & (1u << sec))) continue; + /* First pass: is there anything to show, and how wide is it */ size_t aW[C_COUNT]; for(int c = 0; c < C_COUNT; ++c) aW[c] = strlen(aHdr[c]); - for(size_t u = 0; u < pS->uSet; ++u){ - const DasSettable* p = pS->aSet + u; - if(p->sec != sec) continue; - const char* sShort = strchr(p->sKey, '.') ? strchr(p->sKey, '.') + 1 : p->sKey; - DasSettable_values(p, sVals, sizeof(sVals)); - size_t a[C_COUNT] = { - strlen(sShort), strlen(DasSettable_type(p)), strlen(DasSettable_units(p)), - strlen(p->sDefault), 3, strlen(sVals), 0 - }; - for(int c = 0; c < C_DESC; ++c) if(a[c] > aW[c]) aW[c] = a[c]; + size_t uRows = 0; + for(size_t g = 0; g < DasSrc_groupCount(pSrc, sec); ++g){ + const DasSrcGroup* pG = DasSrc_group(pSrc, sec, g); + for(size_t i = 0; i < DasSrcGroup_propCount(pG); ++i){ + const DasSrcProp* p = DasSrcGroup_prop(pG, i); + if((DasSrcProp_ctrl(p) == NULL) && !g_bVerbose) continue; + ++uRows; + const char* sKey = DasSrcProp_key(p); + const char* sShort = strchr(sKey, '.') ? strchr(sKey, '.') + 1 : sKey; + const char* sDef = DasSrcProp_value(p); + propValues(p, sVals, sizeof(sVals)); + size_t a[C_COUNT] = { + strlen(sShort), strlen(propTypeName(p)), strlen(DasSrcProp_units(p)), + sDef ? strlen(sDef) : 0, DasSrcProp_ctrl(p) ? 3 : 5, strlen(sVals), 0 + }; + for(int c = 0; c < C_DESC; ++c) if(a[c] > aW[c]) aW[c] = a[c]; + } } + if(uRows == 0) continue; int nDescCol = 2; for(int c = 0; c < C_DESC; ++c) nDescCol += (int)aW[c] + 2; - fprintf(pOut, "\n%s:\n", g_sSecName[sec]); + fprintf(pOut, "\n%s:\n", das_src_sec_name(sec)); fprintf(pOut, " %-*s %-*s %-*s %-*s %-*s %-*s %s\n", (int)aW[C_KEY], aHdr[C_KEY], (int)aW[C_TYPE], aHdr[C_TYPE], (int)aW[C_UNITS], aHdr[C_UNITS], (int)aW[C_DEF], aHdr[C_DEF], @@ -1571,53 +730,102 @@ static void DasHttpSrc_describe(const DasHttpSrc* pS, FILE* pOut, const char* sL } fputs("\n", pOut); - for(size_t u = 0; u < pS->uSet; ++u){ - const DasSettable* p = pS->aSet + u; - if(p->sec != sec) continue; - const char* sShort = strchr(p->sKey, '.') ? strchr(p->sKey, '.') + 1 : p->sKey; - DasSettable_values(p, sVals, sizeof(sVals)); - fprintf(pOut, " %-*s %-*s %-*s %-*s %-*s %-*s ", - (int)aW[C_KEY], sShort, (int)aW[C_TYPE], DasSettable_type(p), - (int)aW[C_UNITS], DasSettable_units(p), (int)aW[C_DEF], p->sDefault, - (int)aW[C_REQ], p->bRequired ? "yes" : "", (int)aW[C_VALS], sVals); - fputsWrapped(DasSettable_desc(p, sDesc, sizeof(sDesc)), nDescCol, pOut); - fputc('\n', pOut); + for(size_t g = 0; g < DasSrc_groupCount(pSrc, sec); ++g){ + const DasSrcGroup* pG = DasSrc_group(pSrc, sec, g); + for(size_t i = 0; i < DasSrcGroup_propCount(pG); ++i){ + const DasSrcProp* p = DasSrcGroup_prop(pG, i); + const DasSrcCtrl* pC = DasSrcProp_ctrl(p); + if((pC == NULL) && !g_bVerbose) continue; + const char* sKey = DasSrcProp_key(p); + const char* sShort = strchr(sKey, '.') ? strchr(sKey, '.') + 1 : sKey; + const char* sDef = DasSrcProp_value(p); + const char* sReq = (pC == NULL) ? "fixed" : DasSrcCtrl_required(pC) ? "yes" : ""; + propValues(p, sVals, sizeof(sVals)); + if(pC == NULL) fputs(dimOn(), pOut); + fprintf(pOut, " %-*s %-*s %-*s %-*s %-*s %-*s ", + (int)aW[C_KEY], sShort, (int)aW[C_TYPE], propTypeName(p), + (int)aW[C_UNITS], DasSrcProp_units(p), (int)aW[C_DEF], sDef ? sDef : "", + (int)aW[C_REQ], sReq, (int)aW[C_VALS], sVals); + fputsWrapped(DasSrcProp_title(p, sTitle, sizeof(sTitle)), nDescCol, pOut); + if(pC == NULL) fputs(dimOff(), pOut); + fputc('\n', pOut); + } } } - /* Usage hints are for the first dozen runs, not the hundredth. They - live in 'data -h' and come back with -v. */ - if(!g_bVerbose) return; +} - /* Shortcut hints specific to this source */ +/* Shortcut hints specific to this source: the coordinates that take a + * MIN,MAX[,STEP] range */ +static void printShortcuts(const DasSrc* pSrc, FILE* pOut) +{ fputs("\nShortcuts:\n", pOut); - char sLastGroup[NAME_SZ] = {'\0'}; - for(size_t u = 0; u < pS->uSet; ++u){ - const DasSettable* p = pS->aSet + u; - if((p->sec != SEC_COORD)||strieq(p->sGroup, sLastGroup)) continue; - strcpyz(sLastGroup, p->sGroup, NAME_SZ); - bool bMin = DasHttpSrc_hasProp(pS, SEC_COORD, p->sGroup, "min"); - bool bMax = DasHttpSrc_hasProp(pS, SEC_COORD, p->sGroup, "max"); - if(!bMin || !bMax) continue; - const char* sStep = DasHttpSrc_hasProp(pS, SEC_COORD, p->sGroup, "res") ? "RES" : - DasHttpSrc_hasProp(pS, SEC_COORD, p->sGroup, "inter") ? "INTER" : NULL; - fprintf(pOut, " %s=MIN,MAX%s%s\n", p->sGroup, sStep ? "," : "", sStep ? sStep : ""); + for(size_t g = 0; g < DasSrc_groupCount(pSrc, das_src_sec_coord); ++g){ + const DasSrcGroup* pG = DasSrc_group(pSrc, das_src_sec_coord, g); + const DasSrcProp* pMin = DasSrcGroup_findProp(pG, "min"); + const DasSrcProp* pMax = DasSrcGroup_findProp(pG, "max"); + if((pMin == NULL)||(pMax == NULL)||(DasSrcProp_ctrl(pMin) == NULL)||(DasSrcProp_ctrl(pMax) == NULL)) + continue; + const DasSrcProp* pRes = DasSrcGroup_findProp(pG, "res"); + const DasSrcProp* pInt = DasSrcGroup_findProp(pG, "inter"); + const char* sStep = (pRes && DasSrcProp_ctrl(pRes)) ? "RES" : + (pInt && DasSrcProp_ctrl(pInt)) ? "INTER" : NULL; + fprintf(pOut, " %s=MIN,MAX%s%s\n", DasSrcGroup_name(pG), sStep ? "," : "", sStep ? sStep : ""); } prnPara(pOut, 2, "KEY alone sets a boolean key true, N alone starts from example N, and the " "section prefix (coords., data., ...) may be dropped when unambiguous. " "See '" PROG " data -h'."); } +static void describeSrc(const DasSrc* pSrc, FILE* pOut, const char* sUri) +{ + const char* sVer = DasSrc_version(pSrc); + fprintf(pOut, "HttpStreamSrc%s%s: %s\n", sVer[0] ? " " : "", sVer, DasSrc_name(pSrc)); + prnField(pOut, "Title:", DasSrc_title(pSrc)); + prnField(pOut, "Summary:", DasJdo_string(DasSrc_jdo(pSrc, D2FRAG_DESC))); + prnField(pOut, "URI:", sUri); + if(g_bVerbose){ + prnField(pOut, "From:", DasSrc_url(pSrc)); + size_t uBases = DasStreamSrc_urlCount(pSrc); + if(uBases > 0){ + fprintf(pOut, "Server: %s", DasStreamSrc_baseUrl(pSrc, 0)); + if(uBases > 1) fprintf(pOut, " (+%zu alternate)", uBases - 1); + fputs("\n", pOut); + } + } + prnAuth(pSrc, pOut); + prnFormats(pSrc, pOut); + + if(g_uShow & SHOW_EXAMPLES) printExamples(pSrc, pOut); + + size_t uCtrls = 0; + for(size_t u = 0; u < DasSrc_propCount(pSrc); ++u) + if(DasSrcProp_ctrl(DasSrc_prop(pSrc, u)) != NULL) ++uCtrls; + if(uCtrls == 0){ + fputs("\nThis source takes no query parameters.\n", pOut); + if(!g_bVerbose) return; + } + else if(g_bVerbose){ + fputs("\nQuery keys, use as KEY=VALUE with 'data' or 'url':\n", pOut); + } + + printKeyTables(pSrc, pOut); + + /* Usage hints are for the first dozen runs, not the hundredth. They + live in 'data -h' and come back with -v. */ + if(g_bVerbose && (uCtrls > 0)) printShortcuts(pSrc, pOut); +} + /* ************************************************************************* */ /* Listing a catalog */ -static void listCatalog(const DasNode* pNode, FILE* pOut, const char* sLabel, const char* sUri) +static void listCatalog(const DasNode* pNode, FILE* pOut, const char* sUri) { const char* sType = jstr(DasNode_getJdo(pNode, NULL), D2FRAG_TYPE); const char* sVer = DasNode_version(pNode); fprintf(pOut, "%s%s%s: %s\n", sType ? sType : "Node", sVer ? " " : "", sVer ? sVer : "", DasNode_label(pNode) ? DasNode_label(pNode) : ""); prnField(pOut, "Title:", DasNode_title(pNode)); - prnField(pOut, sLabel, sUri); + prnField(pOut, "URI:", sUri); if(g_bVerbose) prnField(pOut, "From:", DasNode_srcUrl(pNode)); size_t uKids = DasNode_subCount(pNode); @@ -1658,14 +866,13 @@ static void listCatalog(const DasNode* pNode, FILE* pOut, const char* sLabel, co /* What to show as a node's name. Every node has a catalog URI, nodes * loaded by location carry a local: one, see D2URI_LOCAL. Catalogs get * their child separator appended so the next step is obvious. */ -static const char* nodeUri( +static void nodeUri( const DasNode* pNode, const char* sFallback, char* sBuf, size_t uLen ){ const char* sUri = DasNode_pathUri(pNode); if((sUri == NULL)||(sUri[0] == '\0')) sUri = sFallback; const char* sSep = DasNode_isCatalog(pNode) ? DasNode_pathSep(pNode) : NULL; snprintf(sBuf, uLen, "%s%s", sUri, sSep ? sSep : ""); - return "URI:"; } /* ************************************************************************* */ @@ -1719,52 +926,92 @@ static bool applyRootArg(DasCat* pCat, const char* sVal, bool bMount) return DasCat_setPrefix(pCat, sVal) == DAS_OKAY; } -/* From a node the user named, get the HttpStreamSrc to query. Containers - * of sources pick a child: 'flex', else the first stream source listed. To - * pick another the user puts its name on the end of the path. */ -static DasNode* pickSource(DasNode* pNode, char* sErr, size_t uErr) +/* Does a child's catalog entry provide any media type in the list? The + list is a set of exact types compared ignoring case, and the bare + wildcard, star slash star, takes anything, including an entry with no + provides list at all. Parameters on the catalog side are cut. */ +static bool stubProvides(const DasJdo* pStub, const char* sAccept) { - if(DasNode_isStreamSrc(pNode)) return pNode; + const DasJdo* pProv = (pStub == NULL) ? NULL : DasJdo_get(pStub, "provides"); + bool bList = (pProv != NULL) && (pProv->type == das_json_type_ary); - if(!DasNode_isCatalog(pNode)){ - snprintf(sErr, uErr, "Node %s is not a data source", DasNode_srcUrl(pNode)); - return NULL; + const char* sWant = sAccept; + while(true){ + sWant += strspn(sWant, ", \t"); + size_t uWant = strcspn(sWant, ", \t"); + if(uWant == 0) return false; + if((uWant == 3) && (strncmp(sWant, "*/*", 3) == 0)) return true; + if(bList){ + for(const das_json_ary_el* pEl = DasJdo_aryFirst(pProv); pEl; pEl = pEl->next){ + const char* sHas = DasJdo_string(pEl->value); + if(sHas == NULL) continue; + size_t uHas = strcspn(sHas, "; \t"); + if((uHas == uWant) && (strncasecmp(sWant, sHas, uWant) == 0)) return true; + } + } + sWant += uWant; } +} +static const DasJdo* childStub(const DasNode* pNode, const char* sName) +{ size_t uKids = DasNode_subCount(pNode); - const char* sPick = NULL; - bool bHasCatalogs = false; + for(size_t u = 0; u < uKids; ++u) + if(strcmp(DasNode_subName(pNode, u), sName) == 0) return DasNode_subStub(pNode, u); + return NULL; +} - for(size_t u = 0; u < uKids; ++u){ - const char* sName = DasNode_subName(pNode, u); - das_node_type_e nType = DasNode_subType(pNode, u); - if((nType == d2node_catalog)||(nType == d2node_spdf_cat)||(nType == d2node_spase_cat)) - bHasCatalogs = true; - if(nType != d2node_stream_src) continue; - if(strieq(sName, "flex")){ sPick = sName; break; } - if(sPick == NULL) sPick = sName; +/* The library lists the children it can load; which one to fetch from + * is this program's call, made on the media types each child provides. + * Without -a it wants das3 streams and settles for anything, two asks. + * An -a list is one ask and a set, so a user who lists two formats gets + * every source that emits either and names one on the path if that is + * several. The chosen child is loaded through the node's own tree so + * credentials and the cache follow. */ +static DasSrc* pickSrc(DasNode* pNode, const char* sAccept) +{ + if(DasNode_isStreamSrc(pNode)) return new_DasSrc_node(pNode); + + const char* aNames[MAX_KIDS]; + size_t uKids = dassrc_loadable(pNode, aNames, MAX_KIDS); + if(uKids == 0){ + fprintf(stderr, PROG ": nothing under %s is a source this program can fetch from\n", + DasNode_pathUri(pNode)); + return NULL; + } + if(uKids > MAX_KIDS) uKids = MAX_KIDS; + + const char* aAsk[2] = { DEF_ACCEPT, "*/*" }; + size_t uAsks = 2; + if(sAccept != NULL){ aAsk[0] = sAccept; uAsks = 1; } + + const char* aPick[MAX_KIDS]; + size_t uPick = 0; + const char* sAsk = NULL; + for(size_t a = 0; (a < uAsks) && (uPick == 0); ++a){ + sAsk = aAsk[a]; + for(size_t k = 0; k < uKids; ++k) + if(stubProvides(childStub(pNode, aNames[k]), sAsk)) aPick[uPick++] = aNames[k]; } - if(sPick == NULL){ - if(bHasCatalogs) - snprintf(sErr, uErr, "This is a catalog, not a data source. Run '" PROG - " node' on it and pick a child"); + if(uPick != 1){ + if(uPick == 0) + fprintf(stderr, PROG ": nothing under %s provides %s, the sources it has are:", DasNode_pathUri(pNode), sAsk); else - snprintf(sErr, uErr, "No HttpStreamSrc children under %s", DasNode_srcUrl(pNode)); + fprintf(stderr, PROG ": %s offers %zu sources for %s, put one on the end of the path:", DasNode_pathUri(pNode), uPick, sAsk); + const char** aShow = (uPick == 0) ? aNames : aPick; + size_t uShow = (uPick == 0) ? uKids : uPick; + for(size_t u = 0; u < uShow; ++u) fprintf(stderr, " %s", aShow[u]); + fputc('\n', stderr); return NULL; } - /* Credentials and agent are inherited from the node's root */ - DasNode* pSub = DasNode_subNode(pNode, sPick, NULL, NULL); + DasNode* pSub = DasNode_subNode(pNode, aPick[0], NULL, NULL); if(pSub == NULL){ - snprintf(sErr, uErr, "Couldn't load source '%s' from %s", sPick, DasNode_srcUrl(pNode)); - return NULL; - } - if(!DasNode_isStreamSrc(pSub)){ - snprintf(sErr, uErr, "Child '%s' loaded but is not an HttpStreamSrc", sPick); + fprintf(stderr, PROG ": couldn't load source '%s' under %s\n", aPick[0], DasNode_pathUri(pNode)); return NULL; } - return pSub; + return new_DasSrc_node(pSub); } /* A path did not resolve. Say whether the policy or the catalog is to @@ -1887,13 +1134,6 @@ static int streamUrl(DasCat* pCat, const char* sUrl, float rTimeout) /* ************************************************************************* */ -static bool isDigits(const char* s) -{ - if((s == NULL)||(*s == '\0')) return false; - for(; *s; ++s) if(!isdigit((unsigned char)*s)) return false; - return true; -} - /* Columns available on stdout, 0 when it is not a terminal */ static int termWidth(void) { @@ -1918,7 +1158,7 @@ static bool parseShow(const char* sList, unsigned* puShow) strcpyz(sCopy, sList, sizeof(sCopy)); *puShow = 0; for(char* pTok = strtok(sCopy, ","); pTok != NULL; pTok = strtok(NULL, ",")){ - sec_e sec; + das_src_sec_e sec; if(strieq(pTok, "all")) *puShow = SHOW_ALL; else if(strieq(pTok, "examples")||strieq(pTok, "example")||strieq(pTok, "ex")) *puShow |= SHOW_EXAMPLES; else if(secAlias(pTok, &sec)) *puShow |= (1u << sec); @@ -1940,17 +1180,18 @@ int main(int argc, char** argv) const char* sPath = NULL; const char* sCacheArg = NULL; const char* sRootArg = NULL; - const char* sMountArg = NULL; + const char* aMountArg[MAX_MOUNTS]; + size_t uMountArg = 0; + const char* sAccept = NULL; /* -a, else the two step default in pickSrc */ bool bHelp = false; bool bFresh = false; bool bQuiet = false; - const char* aKeyVals[MAX_SETTABLES]; + const char* aKeyVals[MAX_KEYS]; /* KEY=VALUE arguments, applied in order */ size_t uKeyVals = 0; - DasHttpParam aRaw[MAX_RAW]; + const char* aRaw[MAX_RAW]; /* -p PARAM=VALUE arguments */ size_t uRaw = 0; - int nExample = -1; + int nExample = DASSRC_BLANK_REQ; float rTimeout = 10.0; - char sErr[ERR_SZ] = {'\0'}; for(int i = 1; i < argc; ++i){ const char* sArg = argv[i]; @@ -1966,7 +1207,7 @@ int main(int argc, char** argv) if(strcmp(sArg, "--mount") == 0) sArg = "-m"; if((strcmp(sArg, "-r") == 0)||(strcmp(sArg, "-m") == 0)||(strcmp(sArg, "-s") == 0)|| (strcmp(sArg, "-C") == 0)||(strcmp(sArg, "-p") == 0)||(strcmp(sArg, "-t") == 0)|| - (strcmp(sArg, "-l") == 0)){ + (strcmp(sArg, "-l") == 0)||(strcmp(sArg, "-a") == 0)){ if(i + 1 >= argc){ fprintf(stderr, PROG ": option %s needs a value, use -h for help\n", sArg); return RET_USAGE; @@ -1974,7 +1215,14 @@ int main(int argc, char** argv) const char* sVal = argv[++i]; switch(sArg[1]){ case 'r': sRootArg = sVal; break; - case 'm': sMountArg = sVal; break; + case 'm': + if(uMountArg >= MAX_MOUNTS){ + fprintf(stderr, PROG ": at most %d mounts\n", MAX_MOUNTS); + return RET_USAGE; + } + aMountArg[uMountArg++] = sVal; + break; + case 'a': sAccept = sVal; break; case 's': if(!parseShow(sVal, &g_uShow)){ fprintf(stderr, PROG ": -s takes a list from coords, data, formats, options, " @@ -1995,12 +1243,7 @@ int main(int argc, char** argv) fprintf(stderr, PROG ": -p needs PARAM=VALUE (at most %d of them)\n", MAX_RAW); return RET_USAGE; } - size_t uName = (size_t)(pEq - sVal); - if(uName >= NAME_SZ) uName = NAME_SZ - 1; - memcpy(aRaw[uRaw].sName, sVal, uName); - aRaw[uRaw].sName[uName] = '\0'; - strcpyz(aRaw[uRaw].sVal, pEq + 1, VAL_SZ); - ++uRaw; + aRaw[uRaw++] = sVal; break; } } @@ -2015,7 +1258,7 @@ int main(int argc, char** argv) an example index, not a path. */ if(sCmd == NULL){ sCmd = sArg; continue; } if((sPath == NULL) && (strchr(sArg, '=') == NULL) && !isDigits(sArg)){ sPath = sArg; continue; } - if(uKeyVals < MAX_SETTABLES) aKeyVals[uKeyVals++] = sArg; + if(uKeyVals < MAX_KEYS) aKeyVals[uKeyVals++] = sArg; } if(bHelp){ prnHelp(sCmd); return RET_OK; } g_nCols = termWidth(); @@ -2042,11 +1285,11 @@ int main(int argc, char** argv) const char* sEnv = getenv(ROOT_ENV); if((sRootArg == NULL) && (sEnv != NULL) && (sEnv[0] != '\0')) sRootArg = sEnv; sEnv = getenv(MOUNT_ENV); - if((sMountArg == NULL) && (sEnv != NULL) && (sEnv[0] != '\0')) sMountArg = sEnv; + if((uMountArg == 0) && (sEnv != NULL) && (sEnv[0] != '\0')) aMountArg[uMountArg++] = sEnv; if((sRootArg != NULL) && !applyRootArg(pCat, sRootArg, false)) return finish(pCat, RET_USAGE); - if((sMountArg != NULL) && !applyRootArg(pCat, sMountArg, true)) - return finish(pCat, RET_USAGE); + for(size_t u = 0; u < uMountArg; ++u) + if(!applyRootArg(pCat, aMountArg[u], true)) return finish(pCat, RET_USAGE); if(sCmd == NULL){ fprintf(stderr, PROG ": no command given, use -h for help\n"); @@ -2103,11 +1346,10 @@ int main(int argc, char** argv) DasNode* pNode = DasCat_resolve(pCat, sPath); if(pNode == NULL){ explainUnresolved(pCat, sPath); - return finish(pCat, RET_NODE); + return finish(pCat, RET_NODE); } - const char* sFullUri = DasCat_lastUri(pCat); char sShow[1200]; - const char* sLabel = nodeUri(pNode, sFullUri, sShow, sizeof(sShow)); + nodeUri(pNode, DasCat_lastUri(pCat), sShow, sizeof(sShow)); int nRet = RET_OK; @@ -2121,7 +1363,7 @@ int main(int argc, char** argv) char* sPretty = (char*)DasJdo_writePretty(pJdo, " ", "\n", NULL); if(sPretty){ fputs(sPretty, stdout); fputs("\n", stdout); free(sPretty); } } - return finish(pCat, nRet); + return finish(pCat, nRet); } if(bNode){ @@ -2133,83 +1375,62 @@ int main(int argc, char** argv) type can be fetched from, so it is verbose only for now. */ das_node_type_e nType = DasNode_type(pNode); bool bSrcSet = (nType == d2node_srcset)||(nType == d2node_collection); - if(!bSrcSet || g_bVerbose) listCatalog(pNode, stdout, sLabel, sShow); + if(!bSrcSet || g_bVerbose) listCatalog(pNode, stdout, sShow); if(bSrcSet){ - DasNode* pSrc = pickSource(pNode, sErr, sizeof(sErr)); + DasSrc* pSrc = pickSrc(pNode, sAccept); if(pSrc == NULL){ - daslog_error_v("%s", sErr); + /* pickSrc said why; the listing shows what to name */ + if(!g_bVerbose) listCatalog(pNode, stdout, sShow); nRet = RET_NODE; } + else if(!g_bVerbose){ + describeSrc(pSrc, stdout, sShow); + } else{ - DasHttpSrc src; - if(!DasHttpSrc_init(&src, pSrc, sErr, sizeof(sErr))){ - daslog_error_v("%s", sErr); - nRet = RET_NODE; - } - else if(!g_bVerbose){ - DasHttpSrc_describe(&src, stdout, sLabel, sShow); - } - else{ - /* The child's name is the tail of its path URI, after the - separator this container uses */ - const char* sSrcUri = DasNode_pathUri(pSrc); - const char* sSep = DasNode_pathSep(pNode); - const char* sName = sSrcUri; - if(sSrcUri && sSep && sSep[0]){ - const char* p = strstr(sSrcUri, sSep); - while(p){ sName = p + strlen(sSep); p = strstr(sName, sSep); } - } - char sNote[256]; - snprintf(sNote, sizeof(sNote), "Default source: %s", sName ? sName : "first"); - fputc('\n', stdout); - prnPara(stdout, 0, sNote); - fputc('\n', stdout); - char sSrcShow[1200]; - const char* sSrcLabel = nodeUri(pSrc, sSrcUri ? sSrcUri : "", sSrcShow, sizeof(sSrcShow)); - DasHttpSrc_describe(&src, stdout, sSrcLabel, sSrcShow); + /* The child's name is the tail of its path URI, after the + separator this container uses */ + const char* sSrcUri = DasSrc_path(pSrc); + const char* sSep = DasNode_pathSep(pNode); + const char* sName = sSrcUri; + if(sSep && sSep[0]){ + const char* p = strstr(sSrcUri, sSep); + while(p){ sName = p + strlen(sSep); p = strstr(sName, sSep); } } + char sNote[256]; + snprintf(sNote, sizeof(sNote), "Default source: %s", sName); + fputc('\n', stdout); + prnPara(stdout, 0, sNote); + fputc('\n', stdout); + describeSrc(pSrc, stdout, sSrcUri); } + del_DasSrc(pSrc); } } else if(DasNode_isStreamSrc(pNode)){ - DasHttpSrc src; - if(!DasHttpSrc_init(&src, pNode, sErr, sizeof(sErr))){ - daslog_error_v("%s", sErr); + DasSrc* pSrc = new_DasSrc_node(pNode); + if(pSrc == NULL){ nRet = RET_NODE; } - else - DasHttpSrc_describe(&src, stdout, sLabel, sShow); + else{ + describeSrc(pSrc, stdout, sShow); + del_DasSrc(pSrc); + } } else{ const char* sType = jstr(DasNode_getJdo(pNode, NULL), D2FRAG_TYPE); printf("%s: %s\n%-10s%s\n", sType ? sType : "Node", - DasNode_label(pNode) ? DasNode_label(pNode) : "", sLabel, sShow); + DasNode_label(pNode) ? DasNode_label(pNode) : "", "URI:", sShow); if(g_bVerbose) printf("From: %s\n", DasNode_srcUrl(pNode)); printf("\n" PROG " has no detailed printer for this node type, try the json command.\n"); } - return finish(pCat, nRet); - } - - /* url or data */ - DasNode* pSrc = pickSource(pNode, sErr, sizeof(sErr)); - if(pSrc == NULL){ - daslog_error_v("%s", sErr); - return finish(pCat, RET_NODE); + return finish(pCat, nRet); } - DasHttpSrc src; - if(!DasHttpSrc_init(&src, pSrc, sErr, sizeof(sErr))){ - daslog_error_v("%s", sErr); - return finish(pCat, RET_NODE); - } - - /* Order matters: example first, then explicit keys override it, and -p - raw parameters have the last word. A 0.6 example is raw protocol - parameters, those go straight into the output list before the keys - are translated on top of them. */ - DasHttpParam aExRaw[MAX_RAW]; - size_t uExRaw = 0; + /* url or data. The source outlives nothing here, but it does not need + the tree either, so the order of teardown does not matter. */ + DasSrc* pSrc = pickSrc(pNode, sAccept); + if(pSrc == NULL) return finish(pCat, RET_NODE); /* Example selectors among the keys: examples.N, or bare N as the shortcut, the same pairing as data.sprt and sprt. They are pulled out first since @@ -2227,7 +1448,8 @@ int main(int argc, char** argv) if(bSec){ if(!isDigits(pDot + 1)){ daslog_error_v("Examples are numbered, there is no '%s'", sArg); - return finish(pCat, RET_QUERY); + del_DasSrc(pSrc); + return finish(pCat, RET_QUERY); } sNum = pDot + 1; } @@ -2240,23 +1462,18 @@ int main(int argc, char** argv) /* Nothing asked for at all: run the first example so there is always a quick way to see a source work, and say so */ - if((nExample < 0) && (uKeyVals == 0) && (uRaw == 0) && (DasHttpSrc_exampleCount(&src) > 0)){ + if((nExample < 0) && (uKeyVals == 0) && (uRaw == 0) && (DasSrc_exampleCount(pSrc) > 0)){ nExample = 0; if(!bQuiet) - fprintf(stderr, PROG ": no query given, using example 0, \"%s\"\n", DasHttpSrc_exampleLabel(&src, 0)); + fprintf(stderr, PROG ": no query given, using example 0, \"%s\"\n", DasSrc_exampleLabel(pSrc, 0)); } - if(nExample >= 0){ - if(!DasHttpSrc_applyExample(&src, (size_t)nExample, aExRaw, &uExRaw, MAX_RAW, sErr, sizeof(sErr))){ - daslog_error_v("%s", sErr); - return finish(pCat, RET_QUERY); - } - } - for(size_t u = 0; u < uExRaw; ++u){ - if(!DasHttpSrc_outSet(&src, aExRaw[u].sName, aExRaw[u].sVal, sErr, sizeof(sErr))){ - daslog_error_v("%s", sErr); - return finish(pCat, RET_QUERY); - } + /* Order matters: the example is the base, explicit keys override it, and + -p raw parameters have the last word when the URL is built. */ + DasReq* pReq = DasSrc_newReq(pSrc, nExample); + if(pReq == NULL){ + del_DasSrc(pSrc); + return finish(pCat, RET_QUERY); } for(size_t u = 0; u < uKeyVals; ++u){ char sKey[KEY_SZ]; @@ -2272,54 +1489,48 @@ int main(int argc, char** argv) else{ strcpyz(sKey, aKeyVals[u], KEY_SZ); } - if(!DasHttpSrc_set(&src, sKey, sVal, sErr, sizeof(sErr))){ - daslog_error_v("%s", sErr); - return finish(pCat, RET_QUERY); - } - } - - if(!DasHttpSrc_build(&src, sErr, sizeof(sErr))){ - daslog_error_v("%s", sErr); - return finish(pCat, RET_QUERY); + DasReq_set(pReq, sKey, sVal); /* only an empty key fails, and none is */ } for(size_t u = 0; u < uRaw; ++u){ - if(!DasHttpSrc_outSet(&src, aRaw[u].sName, aRaw[u].sVal, sErr, sizeof(sErr))){ - daslog_error_v("%s", sErr); - return finish(pCat, RET_QUERY); - } - } - if(!DasHttpSrc_checkRequired(&src, sErr, sizeof(sErr))){ - daslog_error_v("%s", sErr); - return finish(pCat, RET_QUERY); - } - + char sName[KEY_SZ]; + const char* pEq = strchr(aRaw[u], '='); /* present, main checked */ + size_t uLen = (size_t)(pEq - aRaw[u]); + if(uLen >= KEY_SZ) uLen = KEY_SZ - 1; + memcpy(sName, aRaw[u], uLen); + sName[uLen] = '\0'; + DasReq_setRaw(pReq, sName, pEq + 1); + } + + /* The library logs the first bad entry, a missing required parameter, + or a source with nothing to query, so a failed URL needs no more + words here. Too small a buffer is the one case that is ours. */ char sUrl[URL_SZ]; if(bUrl){ - if(!DasHttpSrc_url(&src, 0, sUrl, sizeof(sUrl))){ - daslog_error("Query URL too long or no base URL"); - nRet = RET_QUERY; - } - else{ - printf("%s\n", sUrl); - } - return finish(pCat, nRet); + size_t uNeed = DasStreamSrc_url(pSrc, pReq, 0, sUrl, sizeof(sUrl)); + if(uNeed == 0) nRet = RET_QUERY; + else if(uNeed >= sizeof(sUrl)){ daslog_error("Query URL too long"); nRet = RET_QUERY; } + else printf("%s\n", sUrl); + del_DasReq(pReq); + del_DasSrc(pSrc); + return finish(pCat, nRet); } /* data: try each base URL in turn until one answers. A server that - answers with an error is an answer, only connection failures move on */ - size_t uBases = DasHttpSrc_baseCount(&src); - nRet = RET_SERVER; - for(size_t u = 0; u < uBases; ++u){ - if(!DasHttpSrc_url(&src, u, sUrl, sizeof(sUrl))){ - daslog_error("Query URL too long"); - nRet = RET_QUERY; - break; - } + answers with an error is an answer, only connection failures move on. + Base 0 is always asked for, so a source with no base URLs at all is + reported by the library like any other unusable one. */ + size_t uBases = DasStreamSrc_urlCount(pSrc); + for(size_t u = 0; ; ++u){ + size_t uNeed = DasStreamSrc_url(pSrc, pReq, u, sUrl, sizeof(sUrl)); + if(uNeed == 0){ nRet = RET_QUERY; break; } + if(uNeed >= sizeof(sUrl)){ daslog_error("Query URL too long"); nRet = RET_QUERY; break; } nRet = streamUrl(pCat, sUrl, rTimeout); if((nRet != RET_SERVER)||(u + 1 >= uBases)) break; - daslog_warn_v("Trying alternate server %s", DasHttpSrc_baseUrl(&src, u + 1)); + daslog_warn_v("Trying alternate server %s", DasStreamSrc_baseUrl(pSrc, u + 1)); } + del_DasReq(pReq); + del_DasSrc(pSrc); finish(pCat, nRet); das_finish(); return nRet; From 0b56278b595adde0d85472881960dd159ec705c6 Mon Sep 17 00:00:00 2001 From: cpiker Date: Wed, 28 Oct 2026 00:38:39 -0500 Subject: [PATCH 11/18] catalog: move node network based function This is in preparation to adapt catalog code for clients that supply thier own networking such as javascript. Developed-with: Claude Fable 5.1 --- das3/catalog.c | 633 ++++++++++++++++++++++++++++++++++++++++++++ das3/node.c | 672 ++--------------------------------------------- das3/node.h | 22 ++ das3/node_priv.h | 48 +++- 4 files changed, 717 insertions(+), 658 deletions(-) diff --git a/das3/catalog.c b/das3/catalog.c index e59cd66..2e406ff 100644 --- a/das3/catalog.c +++ b/das3/catalog.c @@ -25,12 +25,18 @@ #define _POSIX_C_SOURCE 200112L +#include #include #include #include +#include +#include +#include +#include #include "util.h" #include "log.h" +#include "http.h" #include "node_priv.h" #include "source.h" #include "catalog.h" @@ -69,6 +75,633 @@ struct das_cat { char* sLastUri; /* that path as the policy understood it */ }; +/* ************************************************************************* */ +/* Loading nodes: the native driver + * + * Everything below, down to the small helpers, is what node.c does not do: + * read a file, consult the node cache, make an HTTP request, and walk a + * catalog tree by loading one node after another. node.c only parses + * bytes it is handed and reads the result. A client with its own + * transport, a browser build in particular, uses node.h alone and never + * links this file. Moved here from node.c as one piece; the recursive + * walk is replaced by DasNodeWalk in the next step. Until then the + * public pull constructors in node.h are implemented here. */ + +/* ************************************************************************* */ +/* Local file nodes. + * + * Catalog nodes are not only served over HTTP. Test fixtures, detached + * catalogs under development, and 'file://' URLs handed over by other + * tools all want a node read straight from disk. Anything that is not an http(s) URL + * is treated as a path; the "file://" prefix is accepted and stripped. + * Only the path forms "file:///abs/path" and "file://rel/path" are handled, + * host names in file URLs are not. */ + +static bool _DasNode_isFileUrl(const char* sUrl) +{ + if(sUrl == NULL) return false; + if(strncmp(sUrl, "http://", 7) == 0) return false; + if(strncmp(sUrl, "https://", 8) == 0) return false; + return true; +} + +static DasAry* _DasNode_readFile(const char* sUrl) +{ + const char* sPath = sUrl; + if(strncmp(sUrl, "file://", 7) == 0) sPath += 7; + + FILE* pFile = fopen(sPath, "rb"); + if(pFile == NULL){ + daslog_warn_v("Couldn't open node file %s, %s", sPath, strerror(errno)); + return NULL; + } + + DasAry* pAry = new_DasAry("node_file", vtUByte, 1, NULL, RANK_1(0), UNIT_DIMENSIONLESS); + char buf[16384]; + size_t uRead = 0; + while((uRead = fread(buf, 1, sizeof(buf), pFile)) > 0){ + if(!DasAry_append(pAry, (const ubyte*)buf, uRead)){ + dec_DasAry(pAry); + fclose(pFile); + return NULL; + } + } + bool bErr = (ferror(pFile) != 0); + fclose(pFile); + if(bErr){ + daslog_warn_v("Error reading node file %s", sPath); + dec_DasAry(pAry); + return NULL; + } + return pAry; +} + +/* ************************************************************************* */ +/* Node cache */ + +/* Only a DasCat has cache settings, nodes made through the public + * constructors in node.h carry NULL and are never cached. One knob, per + * session, instead of a second process wide one to confuse it with. */ + +/* Cache file name for a URL: the URL minus its scheme with everything + * outside [A-Za-z0-9.-] turned into '_', then a hash of the whole URL so + * that two URLs which sanitize alike still get separate files. Readable + * names make an 'ls' of the cache directory useful when debugging a + * catalog, a bare hash would not. */ +static bool _DasNode_cachePath( + const DasNodeCache* pCache, const char* sUrl, char* sPath, size_t uLen +){ + if((pCache == NULL)||(pCache->sDir[0] == '\0')) return false; + + uint32_t uHash = 2166136261u; /* FNV-1a, no need for anything stronger */ + for(const char* p = sUrl; *p != '\0'; ++p){ + uHash ^= (unsigned char)(*p); + uHash *= 16777619u; + } + + const char* p = sUrl; + if(strncmp(p, "https://", 8) == 0) p += 8; + else if(strncmp(p, "http://", 7) == 0) p += 7; + + char sName[160]; + size_t u = 0; + for(; (*p != '\0') && (u < sizeof(sName) - 1); ++p){ + char c = *p; + sName[u++] = (isalnum((unsigned char)c)||(c == '.')||(c == '-')) ? c : '_'; + } + sName[u] = '\0'; + + int n = snprintf(sPath, uLen, "%s%s%s_%08" PRIx32 ".json", pCache->sDir, DAS_DSEPS, sName, uHash); + return (n > 0) && ((size_t)n < uLen); +} + +/* A fresh enough cached copy, or NULL */ +static DasAry* _DasNode_readCache(const DasNodeCache* pCache, const char* sPath) +{ + if((pCache == NULL)||(pCache->nMaxAgeSec <= 0)) return NULL; + struct stat st; + if(stat(sPath, &st) != 0) return NULL; + if((time(NULL) - st.st_mtime) > pCache->nMaxAgeSec) return NULL; + return _DasNode_readFile(sPath); +} + +/* Written to a side file then renamed, so a reader never sees a partial + * document when two programs walk the catalog at once. Failures only cost + * the cache, so they are logged at debug level and otherwise ignored. */ +static void _DasNode_writeCache(const char* sPath, const char* pBytes, size_t uBytes) +{ + char sTmp[600]; + if(snprintf(sTmp, sizeof(sTmp), "%s.%ld.tmp", sPath, (long)time(NULL)) >= (int)sizeof(sTmp)) return; + + if(das_mkdirsto(sPath) != DAS_OKAY){ + daslog_debug_v("Couldn't create node cache directory for %s", sPath); + return; + } + FILE* pFile = fopen(sTmp, "wb"); + if(pFile == NULL){ + daslog_debug_v("Couldn't write node cache file %s, %s", sTmp, strerror(errno)); + return; + } + bool bOk = (fwrite(pBytes, 1, uBytes, pFile) == uBytes); + bOk = (fclose(pFile) == 0) && bOk; + if(!bOk || (das_rename(sTmp, sPath) != 0)){ + daslog_debug_v("Couldn't finish node cache file %s", sPath); + remove(sTmp); + } +} + +/* The validators a cached node came with live beside it in a .hdr side + * file, at most two lines, the ETag and Last-Modified header values + * verbatim. When the copy is older than the session trusts both go back + * as If-None-Match and If-Modified-Since, and a 304 renews the copy for + * one round trip and no body. Both are kept because servers differ: the + * das2.org tree sends both, GitHub raw sends ETag only, dasflex today + * sends neither. The date is never derived from the cache file's own + * mtime, which renewal moves forward. */ +static void _DasNode_hdrPath(const char* sCache, char* sHdr, size_t uLen) +{ + snprintf(sHdr, uLen, "%s.hdr", sCache); +} + +/* Request header lines for the cached copy at sCache, "" if it has none */ +static void _DasNode_readValidators(const char* sCache, char* sReq, size_t uLen) +{ + sReq[0] = '\0'; + char sHdr[600]; + _DasNode_hdrPath(sCache, sHdr, sizeof(sHdr)); + FILE* pFile = fopen(sHdr, "r"); + if(pFile == NULL) return; + + char sLine[1024]; + size_t u = 0; + while(fgets(sLine, sizeof(sLine), pFile) != NULL){ + size_t uLine = strlen(sLine); + while((uLine > 0) && ((sLine[uLine-1] == '\n')||(sLine[uLine-1] == '\r'))) + sLine[--uLine] = '\0'; + const char* sName = NULL; + const char* sVal = NULL; + if(strncmp(sLine, "ETag: ", 6) == 0){ sName = "If-None-Match"; sVal = sLine + 6; } + else if(strncmp(sLine, "Last-Modified: ", 15) == 0){ sName = "If-Modified-Since"; sVal = sLine + 15; } + if((sName == NULL)||(sVal[0] == '\0')) continue; + int n = snprintf(sReq + u, uLen - u, "%s: %s\r\n", sName, sVal); + if((n < 0)||((size_t)n >= uLen - u)){ sReq[u] = '\0'; break; } + u += (size_t)n; + } + fclose(pFile); +} + +/* Save the validators a 200 carried, or drop the side file if it carried + * none, so a validator from an older body is never sent for a newer one */ +static void _DasNode_writeValidators(const char* sCache, const DasHttpResp* pRes) +{ + char sHdr[600]; + _DasNode_hdrPath(sCache, sHdr, sizeof(sHdr)); + char sETag[256] = {'\0'}; + char sMod[128] = {'\0'}; + bool bETag = DasHttpResp_header(pRes, "ETag", sETag, sizeof(sETag)); + bool bMod = DasHttpResp_header(pRes, "Last-Modified", sMod, sizeof(sMod)); + if(!bETag && !bMod){ remove(sHdr); return; } + + FILE* pFile = fopen(sHdr, "w"); + if(pFile == NULL){ + daslog_debug_v("Couldn't write node cache side file %s, %s", sHdr, strerror(errno)); + return; + } + if(bETag) fprintf(pFile, "ETag: %s\n", sETag); + if(bMod) fprintf(pFile, "Last-Modified: %s\n", sMod); + fclose(pFile); +} + +/* make a node from a *specific* URL, does not call das_error functions unless + * a memory allocation error occurs because network operations are assumed to + * fail sometimes. Returns a new node allocated on the heap if the URL + * contained a recognizable DasCatalog object + */ +static DasNode* _DasNode_mkNode( + const char* sUrl, const char* sPathUri, DasCredMngr* pMgr, + const char* sAgent, float rConSec, const DasNodeCache* pCache +){ + /* Save the final URL, may be needed for error messages, may not match + * sUrl due to re-directs */ + char sFinalUrl[512] = {'\0'}; + + DasAry* pBytesAry = NULL; + char sCache[512] = {'\0'}; + bool bFromCache = false; /* copy used as is, nothing to write back */ + bool bRenewed = false; /* server confirmed the copy, rewrite it to move its age */ + DasHttpResp httpRes; + memset(&httpRes, 0, sizeof(DasHttpResp)); + + if(_DasNode_isFileUrl(sUrl)){ + pBytesAry = _DasNode_readFile(sUrl); + strncpy(sFinalUrl, sUrl, sizeof(sFinalUrl) - 1); + } + else if(_DasNode_cachePath(pCache, sUrl, sCache, sizeof(sCache)) && + ((pBytesAry = _DasNode_readCache(pCache, sCache)) != NULL)){ + daslog_debug_v("Node %s read from cache", sUrl); + strncpy(sFinalUrl, sUrl, sizeof(sFinalUrl) - 1); + bFromCache = true; + } + else{ + /* No copy, or one older than the session trusts. With validators + * from the old copy the request is conditional and a 304 renews it + * for no body bytes. If the server cannot be reached at all the old + * copy still beats nothing: a warm cache keeps working offline. The + * 20 MB limit is a sanity check, no catalog node comes near it. */ + char sReqHdrs[1024] = {'\0'}; + bool bHaveOld = (sCache[0] != '\0') && das_isfile(sCache); + if(bHaveOld) _DasNode_readValidators(sCache, sReqHdrs, sizeof(sReqHdrs)); + + pBytesAry = das_http_readUrl_ex( + sUrl, sAgent, pMgr, &httpRes, 1024*1024*20, rConSec, sReqHdrs + ); + if(pBytesAry != NULL){ + das_url_toStr(&(httpRes.url), sFinalUrl, sizeof(sFinalUrl)); + } + else if(httpRes.nCode == 304){ + daslog_debug_v("Node %s unchanged on the server, cached copy renewed", sUrl); + pBytesAry = _DasNode_readFile(sCache); + strncpy(sFinalUrl, sUrl, sizeof(sFinalUrl) - 1); + bRenewed = true; + } + else if(bHaveOld){ + daslog_warn_v("%s, using the cached copy of %s", + httpRes.sError ? httpRes.sError : "Fetch failed", sUrl); + pBytesAry = _DasNode_readFile(sCache); + strncpy(sFinalUrl, sUrl, sizeof(sFinalUrl) - 1); + bFromCache = true; + } + else{ + daslog_warn_v("%s", httpRes.sError ? httpRes.sError : "Fetch failed"); + } + } + if(pBytesAry == NULL){ + DasHttpResp_freeFields(&httpRes); + return NULL; + } + + size_t uBytes; + const char* pBytes = (const char*) (DasAry_getBytesIn(pBytesAry, DIM0, &uBytes)); + DasNode* pNode = _DasNode_fromJson(pBytes, uBytes, sUrl, sFinalUrl, sPathUri); + if(pNode != NULL){ pNode->pMgr = pMgr; pNode->sAgent = sAgent; pNode->pCache = pCache; } + + /* Only documents that parse are worth keeping, a server error page is + * not. A renewal rewrites the same bytes, which is what moves the + * copy's age, and keeps the validators it already has. */ + if((pNode != NULL) && (sCache[0] != '\0') && !bFromCache){ + _DasNode_writeCache(sCache, pBytes, uBytes); + if(!bRenewed) _DasNode_writeValidators(sCache, &httpRes); + } + + DasHttpResp_freeFields(&httpRes); + dec_DasAry(pBytesAry); /* Done with the byte array, data in json buffer now */ + return pNode; +} + +/* ************************************************************************* */ +/* Sub Node Construction */ + + +static DasNode* _DasNode_loadSubNode_dasCat( + DasCatNode* pThis, const char* sRelPath, DasCredMngr* pMgr, + const char* sAgent +){ + if(!DasNode_isJson((DasNode*)pThis)){ + daslog_error("XML catalogs not yet supported"); + return NULL; + } + const DasJdo* pDir = (DasJdo*) pThis->pContainer; + + if((pDir == NULL) || (pDir->type != das_json_type_dict)){ + daslog_error_v("Catalog container node missing for node from %s", + pThis->base.sURL); + return NULL; + } + + /* Run over all the keys and see if any of them match the front of the + * requested relative path. If so try all the URLs given for full object + * realization before giving up */ + const das_json_dict_el* pEl = NULL; + size_t uLen; + const DasJdo* pChild = NULL; + const char* sChild = NULL; + const DasJdo* pChildUrls = NULL; + const das_json_ary_el* pUrlEl; + char sSubUri[560] = {'\0'}; + const char* sUrl = NULL; + const char* sSubRelPath = NULL; + DasNode* pNode = NULL; + DasNode* pDecendent = NULL; + int nUrl; + + /* Trying it implement a tree search that will auto backout and try again + * if a URL turns out to be broken, or contains bad data, or is the wrong + * child. */ + for(pEl = DasJdo_dictFirst(pDir); pEl != NULL; pEl = pEl->next){ + sChild = pEl->name->string; + + uLen = strlen(sChild); + + /* See if this child and the requested item have the same chars up to + * the length of the child */ + if(strncmp(sChild, sRelPath, uLen) != 0) continue; + + pChild = pEl->value; /* If this never get's set, then we had no + * children with matching names */ + + /* Just stop using this catalog node if it has broken syntax */ + if((pChild == NULL )||(pChild->type != das_json_type_dict)){ + daslog_error_v("Catalog error @ %s: sub item '%s' is not a dictionary", + pThis->base.sURL, pEl->name->string); + return NULL; + } + + pChildUrls = DasJdo_get(pChild, D2FRAG_URLS); + if((pChildUrls == NULL)||(pChildUrls->type != das_json_type_ary)){ + daslog_error_v("From %s: %s element of node '%s' doesn't " + "have a URLS array", pThis->base.sURL, sChild, + pThis->sContainer); + return NULL; + } + + /* Loop over URLs for child */ + nUrl = -1; + pNode = NULL; + + /* this is a slow time out.. */ + float rConSec = DASHTTP_TO_MIN * DASHTTP_TO_MULTI; + + for(pUrlEl = DasJdo_aryFirst(pChildUrls); pUrlEl != NULL; + pUrlEl = pUrlEl->next){ + pNode = NULL; + nUrl += 1; + sUrl = DasJdo_string(pUrlEl->value); + if(sUrl == NULL){ + daslog_error_v("From %s: %s/%s/%d element is not a string", + pThis->base.sURL, D2FRAG_URLS, sChild, nUrl); + continue; + } + + /* tell sub-item it's name */ + snprintf(sSubUri, 559, "%s%s%s", pThis->base.sPath, pThis->sPathSep, sChild); + + /* TODO: Split mkNode and server contact into two separate functions + * so that we know if we can try again with a slow server, of if + * we just received bad data and shouldn't waste time reading it + * again */ + pNode = _DasNode_mkNode(sUrl, sSubUri, pMgr, sAgent, rConSec, pThis->base.pCache); + if(pNode == NULL) continue; + + /* If after removing the child portion from the relative path there + * is nothing left we're done. */ + sSubRelPath = sRelPath + uLen; + + if(sSubRelPath[0] == '\0'){ + DasAry_append(pThis->pSubNodes, (const ubyte*) &pNode, 1); + DasAry_append(pThis->pSubPaths, (const ubyte*) sChild, strlen(sChild)+1); + DasAry_markEnd(pThis->pSubPaths, DIM1); + return pNode; + } + + /* There's more, see if it's possible for this child to complete the + * path. */ + if(_DasNode_subMatch(pNode, sSubRelPath)){ + + pDecendent = DasNode_subNode((DasNode*)pNode, sSubRelPath, pMgr, sAgent); + if(pDecendent){ + /* Worked okay, cache the child node, but return the descendant + * node (however far down it came from */ + DasAry_append(pThis->pSubNodes, (const ubyte*) &pNode, 1); + DasAry_append(pThis->pSubPaths, (const ubyte*) sChild, strlen(sChild)+1); + DasAry_markEnd(pThis->pSubPaths, DIM1); + + return pDecendent; + } + } + else{ + /* This child can't match, stop checking this URL set*/ + _DasNode_delTree(pNode); + pNode = NULL; + } + } + + /* Must have been a look-alike child (or some other failure), try again */ + _DasNode_delTree(pNode); + pNode = NULL; + } + + + /* A miss at one level is normal while the walk backs out and tries other + * URLs or children, so it is not an error here. The caller knows whether + * the whole path failed and has the user's wording of it. */ + const char* sUri = DasNode_pathUri((DasNode*)pThis); + daslog_info_v( + "Node %s (URI '%s') has no child node that starts with %s", + DasNode_label((DasNode*)pThis), sUri ? sUri : "", sRelPath + ); + + return NULL; +} + +static DasNode* _DasNode_loadSubNode_spdfCat( + DasCatNode* pThis, const char* sRelPath, DasCredMngr* pMgr, const char* sAgent +){ + daslog_error("SPDF catalogs are not yet supported"); + return NULL; +} + +static DasNode* _DasNode_loadSubNode_spaseCat( + DasCatNode* pThis, const char* sRelPath, DasCredMngr* pMgr, const char* sAgent +){ + daslog_error("Spase catalogs are not yet supported"); + return NULL; +} + +DasNode* DasNode_subNode( + DasNode* pThis, const char* sRelPath, DasCredMngr* pMgr, const char* sAgent +){ + /* Credentials and agent flow down from the root unless overridden here */ + if(pMgr == NULL) pMgr = pThis->pMgr; + if(sAgent == NULL) sAgent = pThis->sAgent; + + if((sRelPath == NULL) || (sRelPath[0] == '\0')){ + das_error(DASERR_NODE, "Empty relative path, can't lookup scheme definition"); + return NULL; + } + if(!DasNode_isCatalog(pThis)){ + daslog_error_v("Node %s from %s is a terminating node", DasNode_label(pThis), + DasNode_srcUrl(pThis)); + return NULL; + } + DasCatNode* pCat = (DasCatNode*)pThis; + + if((pCat->sPathSep[0] != '\0')&& + (strncmp(sRelPath, pCat->sPathSep, strlen(pCat->sPathSep)) == 0)){ + sRelPath += strlen(pCat->sPathSep); + } + + /* Search the local child cache first before calling on the inter-tubes */ + const char* sSubPath = NULL; + const char* sSubRelPath = NULL; + ptrdiff_t uChildren = DasAry_lengthIn(pCat->pSubNodes, DIM0); + size_t uLen = 0; + DasNode* pNode = NULL; + DasNode* pRet = NULL; + for(ptrdiff_t u = 0; u < uChildren; ++u){ + sSubPath = (const char*)DasAry_getBytesIn(pCat->pSubPaths, DIM1_AT(u), &uLen); + + /* From the DasAry's point of view, the item length includes the null + * because it has to store the null. But from strncmp's point of view + * we don't count the null because the rel path might not be a terminating + * path (i.e. it may have components below the child */ + if(uLen == 0){ + daslog_warn_v("Node at %s has a zero-length sub-path", pThis->sURL); + continue; + } + else{ + uLen -= 1; + } + + if(strncmp(sSubPath, sRelPath, uLen) == 0){ + + /* This child completes at least part of the subpath */ + pNode = *((DasNode**) DasAry_getAt(pCat->pSubNodes, vtUnknown, IDX0(u))); + + sSubRelPath = sRelPath + uLen; + if(sSubRelPath[0] == '\0'){ + /* actually it completes all of it*/ + return pNode; + } + else{ + /* There's more, see if it's possible for this child to complete + * the path, if not don't fail, just keep looking */ + if( ! _DasNode_subMatch(pNode, sSubRelPath)) continue; + + pRet = DasNode_subNode(pNode, sSubRelPath, pMgr, sAgent); + if(pRet != NULL) return pRet; + } + } + } + + /* I don't have this sub node: Internet! To the rescue! */ + + switch(pThis->nType){ + case d2node_collection: + case d2node_catalog: + case d2node_srcset: + return _DasNode_loadSubNode_dasCat(pCat, sRelPath, pMgr, sAgent); + case d2node_spdf_cat: + return _DasNode_loadSubNode_spdfCat(pCat, sRelPath, pMgr, sAgent); + case d2node_spase_cat: + return _DasNode_loadSubNode_spaseCat(pCat, sRelPath, pMgr, sAgent); + break; + default: + /* Should have been ruled out by constructor, but double check since + * new types may be added */ + das_error(DASERR_ASSERT, "Logic error in libdas2"); return NULL; + } + + return NULL; +} + +/* ************************************************************************* */ +/* Construction */ + +DasNode* _DasNode_newUrl( + const char* sUrl, const char* sPathUri, DasCredMngr* pMgr, const char* sAgent, + const DasNodeCache* pCache +){ + /* Every node has a URI; one loaded on its own is named by where it is */ + char sUri[600]; + if((sPathUri == NULL)||(sPathUri[0] == '\0')) _das_uri_local(sUrl, sUri, sizeof(sUri)); + else _das_uri_expand(sPathUri, sUri, sizeof(sUri)); + + /* Since they want a specific URL, go for the max timeout */ + DasNode* pNode = _DasNode_mkNode(sUrl, sUri, pMgr, sAgent, DASHTTP_TO_MAX, pCache); + if(pNode) pNode->bOwner = true; + return pNode; +} + +DasNode* new_DasNode_url( + const char* sUrl, const char* sPathUri, DasCredMngr* pMgr, const char* sAgent +){ + return _DasNode_newUrl(sUrl, sPathUri, pMgr, sAgent, NULL); +} + +DasNode* _DasNode_new( + const char* sPathUri, DasCredMngr* pMgr, const char* sAgent, + const DasNodeCache* pCache +){ + /* Find the node using the path, start at the root */ + DasNode* pTop = NULL; + DasNode* pNode = NULL; + + /* Expanding shorthands here, in the library, means every client gets + the same spelling. Names under local: belong to nodes loaded by + location in some session; the federated roots have never heard of + them and asking would only cost a round trip. */ + char sFullUri[600] = {'\0'}; + if(sPathUri != NULL){ + sPathUri = _das_uri_expand(sPathUri, sFullUri, sizeof(sFullUri)); + if(_das_uri_isLocal(sPathUri)){ + daslog_debug_v("%s is a local: name, those are not routed through the federated catalog", sPathUri); + return NULL; + } + } + + /* Changing node lookup and time outs. New strategy, try to contact a node + if it doesn't work within the time out, try the next one. If all fail + then increase the time out and try again. + */ + + size_t uRoots = 0; + const char** aRoots = das_root_urls(&uRoots); + float rConSec = DASHTTP_TO_MIN; + + while((rConSec <= DASHTTP_TO_MAX )&&(pNode == NULL)){ + for(size_t i = 0; i < uRoots; ++i){ + /* Credentials should never be needed for top level nodes */ + pTop = _DasNode_mkNode(aRoots[i], NULL, NULL, sAgent, rConSec, pCache); + if(pTop == NULL) continue; + if(pTop->nType != d2node_catalog) continue; + + if(sPathUri == NULL){ /* They may be looking for a root node */ + pNode = pTop; + break; + } + + /* Depends on the fact that top node is a das catalog */ + pNode = _DasNode_loadSubNode_dasCat((DasCatNode*)pTop, sPathUri, pMgr, sAgent); + if(pNode != NULL) break; + } + rConSec *= DASHTTP_TO_MULTI; + } + + if(pNode == NULL){ + daslog_error_v("Failed to load the requested node %s from any of the " + "built in catalog trees", sPathUri); + return NULL; + } + + /* The caller gets this node, so mark it theirs before freeing the walk + * above it; _DasNode_delTree() then leaves it alone even when + * pTop == pNode */ + pNode->bOwner = true; + pNode->pMgr = pMgr; /* pTop was fetched without them, the kept node + hands them to whatever it loads later */ + pNode->sAgent = sAgent; + pNode->pCache = pCache; + _DasNode_delTree(pTop); + + return pNode; +} + +DasNode* new_DasNode(const char* sPathUri, DasCredMngr* pMgr, const char* sAgent) +{ + return _DasNode_new(sPathUri, pMgr, sAgent, NULL); +} + + /* ************************************************************************* */ /* Small helpers */ diff --git a/das3/node.c b/das3/node.c index 5eafd96..aa2623b 100644 --- a/das3/node.c +++ b/das3/node.c @@ -23,29 +23,18 @@ */ #define _POSIX_C_SOURCE 200112L -#include -#include #include #include -#include -#include -#include -#include #ifndef _WIN32 -#include #include #else -#include #include #define getcwd _getcwd #endif -#include - #include "util.h" #include "log.h" -#include "http.h" #include "array.h" #include "node.h" @@ -72,29 +61,6 @@ const char** das_root_urls(size_t* pLen){ } -#define D2C_LOCAL_SUB_SZ 32 - -/* Catalog nodes have extra arrays to cache nodes */ -#define _D2C_SEP_SZ 8 -typedef struct das_catalog { - DasNode base; - - DasAry* pSubNodes; - - DasAry* pSubPaths; - - /* The json object or xml element tag which contains the sub elements */ - void* pContainer; - - /* The path separator, defaults to '/' unless overridden by the object */ - char sPathSep[_D2C_SEP_SZ]; - - /* The name of the element which contains sub elements */ - char sContainer[64]; - -} DasCatNode; - - static das_node_type_e das_node_type(const char* sType){ if(sType == NULL) return d2node_inv; if(strcmp(sType, D2CV_TYPE_CATALOG) == 0) return d2node_catalog; @@ -269,196 +235,6 @@ das_node_type_e DasNode_subType(const DasNode* pThis, size_t uIdx) return das_node_type( DasJdo_string( DasJdo_get(pStub, D2FRAG_TYPE) ) ); } -/* ************************************************************************* */ -/* Local file nodes. - * - * Catalog nodes are not only served over HTTP. Test fixtures, detached - * catalogs under development, and 'file://' URLs handed over by other - * tools all want a node read straight from disk. Anything that is not an http(s) URL - * is treated as a path; the "file://" prefix is accepted and stripped. - * Only the path forms "file:///abs/path" and "file://rel/path" are handled, - * host names in file URLs are not. */ - -static bool _DasNode_isFileUrl(const char* sUrl) -{ - if(sUrl == NULL) return false; - if(strncmp(sUrl, "http://", 7) == 0) return false; - if(strncmp(sUrl, "https://", 8) == 0) return false; - return true; -} - -static DasAry* _DasNode_readFile(const char* sUrl) -{ - const char* sPath = sUrl; - if(strncmp(sUrl, "file://", 7) == 0) sPath += 7; - - FILE* pFile = fopen(sPath, "rb"); - if(pFile == NULL){ - daslog_warn_v("Couldn't open node file %s, %s", sPath, strerror(errno)); - return NULL; - } - - DasAry* pAry = new_DasAry("node_file", vtUByte, 1, NULL, RANK_1(0), UNIT_DIMENSIONLESS); - char buf[16384]; - size_t uRead = 0; - while((uRead = fread(buf, 1, sizeof(buf), pFile)) > 0){ - if(!DasAry_append(pAry, (const ubyte*)buf, uRead)){ - dec_DasAry(pAry); - fclose(pFile); - return NULL; - } - } - bool bErr = (ferror(pFile) != 0); - fclose(pFile); - if(bErr){ - daslog_warn_v("Error reading node file %s", sPath); - dec_DasAry(pAry); - return NULL; - } - return pAry; -} - -/* ************************************************************************* */ -/* Node cache */ - -/* Only a DasCat has cache settings, nodes made through the public - * constructors in node.h carry NULL and are never cached. One knob, per - * session, instead of a second process wide one to confuse it with. */ - -/* Cache file name for a URL: the URL minus its scheme with everything - * outside [A-Za-z0-9.-] turned into '_', then a hash of the whole URL so - * that two URLs which sanitize alike still get separate files. Readable - * names make an 'ls' of the cache directory useful when debugging a - * catalog, a bare hash would not. */ -static bool _DasNode_cachePath( - const DasNodeCache* pCache, const char* sUrl, char* sPath, size_t uLen -){ - if((pCache == NULL)||(pCache->sDir[0] == '\0')) return false; - - uint32_t uHash = 2166136261u; /* FNV-1a, no need for anything stronger */ - for(const char* p = sUrl; *p != '\0'; ++p){ - uHash ^= (unsigned char)(*p); - uHash *= 16777619u; - } - - const char* p = sUrl; - if(strncmp(p, "https://", 8) == 0) p += 8; - else if(strncmp(p, "http://", 7) == 0) p += 7; - - char sName[160]; - size_t u = 0; - for(; (*p != '\0') && (u < sizeof(sName) - 1); ++p){ - char c = *p; - sName[u++] = (isalnum((unsigned char)c)||(c == '.')||(c == '-')) ? c : '_'; - } - sName[u] = '\0'; - - int n = snprintf(sPath, uLen, "%s%s%s_%08" PRIx32 ".json", pCache->sDir, DAS_DSEPS, sName, uHash); - return (n > 0) && ((size_t)n < uLen); -} - -/* A fresh enough cached copy, or NULL */ -static DasAry* _DasNode_readCache(const DasNodeCache* pCache, const char* sPath) -{ - if((pCache == NULL)||(pCache->nMaxAgeSec <= 0)) return NULL; - struct stat st; - if(stat(sPath, &st) != 0) return NULL; - if((time(NULL) - st.st_mtime) > pCache->nMaxAgeSec) return NULL; - return _DasNode_readFile(sPath); -} - -/* Written to a side file then renamed, so a reader never sees a partial - * document when two programs walk the catalog at once. Failures only cost - * the cache, so they are logged at debug level and otherwise ignored. */ -static void _DasNode_writeCache(const char* sPath, const char* pBytes, size_t uBytes) -{ - char sTmp[600]; - if(snprintf(sTmp, sizeof(sTmp), "%s.%ld.tmp", sPath, (long)time(NULL)) >= (int)sizeof(sTmp)) return; - - if(das_mkdirsto(sPath) != DAS_OKAY){ - daslog_debug_v("Couldn't create node cache directory for %s", sPath); - return; - } - FILE* pFile = fopen(sTmp, "wb"); - if(pFile == NULL){ - daslog_debug_v("Couldn't write node cache file %s, %s", sTmp, strerror(errno)); - return; - } - bool bOk = (fwrite(pBytes, 1, uBytes, pFile) == uBytes); - bOk = (fclose(pFile) == 0) && bOk; - if(!bOk || (das_rename(sTmp, sPath) != 0)){ - daslog_debug_v("Couldn't finish node cache file %s", sPath); - remove(sTmp); - } -} - -/* The validators a cached node came with live beside it in a .hdr side - * file, at most two lines, the ETag and Last-Modified header values - * verbatim. When the copy is older than the session trusts both go back - * as If-None-Match and If-Modified-Since, and a 304 renews the copy for - * one round trip and no body. Both are kept because servers differ: the - * das2.org tree sends both, GitHub raw sends ETag only, dasflex today - * sends neither. The date is never derived from the cache file's own - * mtime, which renewal moves forward. */ -static void _DasNode_hdrPath(const char* sCache, char* sHdr, size_t uLen) -{ - snprintf(sHdr, uLen, "%s.hdr", sCache); -} - -/* Request header lines for the cached copy at sCache, "" if it has none */ -static void _DasNode_readValidators(const char* sCache, char* sReq, size_t uLen) -{ - sReq[0] = '\0'; - char sHdr[600]; - _DasNode_hdrPath(sCache, sHdr, sizeof(sHdr)); - FILE* pFile = fopen(sHdr, "r"); - if(pFile == NULL) return; - - char sLine[1024]; - size_t u = 0; - while(fgets(sLine, sizeof(sLine), pFile) != NULL){ - size_t uLine = strlen(sLine); - while((uLine > 0) && ((sLine[uLine-1] == '\n')||(sLine[uLine-1] == '\r'))) - sLine[--uLine] = '\0'; - const char* sName = NULL; - const char* sVal = NULL; - if(strncmp(sLine, "ETag: ", 6) == 0){ sName = "If-None-Match"; sVal = sLine + 6; } - else if(strncmp(sLine, "Last-Modified: ", 15) == 0){ sName = "If-Modified-Since"; sVal = sLine + 15; } - if((sName == NULL)||(sVal[0] == '\0')) continue; - int n = snprintf(sReq + u, uLen - u, "%s: %s\r\n", sName, sVal); - if((n < 0)||((size_t)n >= uLen - u)){ sReq[u] = '\0'; break; } - u += (size_t)n; - } - fclose(pFile); -} - -/* Save the validators a 200 carried, or drop the side file if it carried - * none, so a validator from an older body is never sent for a newer one */ -static void _DasNode_writeValidators(const char* sCache, const DasHttpResp* pRes) -{ - char sHdr[600]; - _DasNode_hdrPath(sCache, sHdr, sizeof(sHdr)); - char sETag[256] = {'\0'}; - char sMod[128] = {'\0'}; - bool bETag = DasHttpResp_header(pRes, "ETag", sETag, sizeof(sETag)); - bool bMod = DasHttpResp_header(pRes, "Last-Modified", sMod, sizeof(sMod)); - if(!bETag && !bMod){ remove(sHdr); return; } - - FILE* pFile = fopen(sHdr, "w"); - if(pFile == NULL){ - daslog_debug_v("Couldn't write node cache side file %s, %s", sHdr, strerror(errno)); - return; - } - if(bETag) fprintf(pFile, "ETag: %s\n", sETag); - if(bMod) fprintf(pFile, "Last-Modified: %s\n", sMod); - fclose(pFile); -} - -/* ************************************************************************* */ -/* Direct Node Construction */ - -/* Build a node from JSON text. sUrl is where it came from and sFinalUrl - * the same after redirects, both for messages. sPathUri may be NULL. */ /* ************************************************************************* */ /* Presenting every document in the current layout * @@ -581,7 +357,10 @@ static bool _DasNode_upgrade(DasJdo* pRoot, das_node_type_e nType, void** ppShim } /* ---- 2018 Collection shim, end ---- */ -static DasNode* _DasNode_fromJson( +/* Build a node from JSON text. sUrl is where it came from and sFinalUrl + * the same after redirects, both for messages. sPathUri may be NULL. The + * node is not marked as owned; constructors decide that. */ +DasNode* _DasNode_fromJson( const char* pBytes, size_t uBytes, const char* sUrl, const char* sFinalUrl, const char* sPathUri ){ @@ -695,102 +474,6 @@ static DasNode* _DasNode_fromJson( return pBase; } -/* make a node from a *specific* URL, does not call das_error functions unless - * a memory allocation error occurs because network operations are assumed to - * fail sometimes. Returns a new node allocated on the heap if the URL - * contained a recognizable DasCatalog object - */ -static DasNode* _DasNode_mkNode( - const char* sUrl, const char* sPathUri, DasCredMngr* pMgr, - const char* sAgent, float rConSec, const DasNodeCache* pCache -){ - /* Save the final URL, may be needed for error messages, may not match - * sUrl due to re-directs */ - char sFinalUrl[512] = {'\0'}; - - DasAry* pBytesAry = NULL; - char sCache[512] = {'\0'}; - bool bFromCache = false; /* copy used as is, nothing to write back */ - bool bRenewed = false; /* server confirmed the copy, rewrite it to move its age */ - DasHttpResp httpRes; - memset(&httpRes, 0, sizeof(DasHttpResp)); - - if(_DasNode_isFileUrl(sUrl)){ - pBytesAry = _DasNode_readFile(sUrl); - strncpy(sFinalUrl, sUrl, sizeof(sFinalUrl) - 1); - } - else if(_DasNode_cachePath(pCache, sUrl, sCache, sizeof(sCache)) && - ((pBytesAry = _DasNode_readCache(pCache, sCache)) != NULL)){ - daslog_debug_v("Node %s read from cache", sUrl); - strncpy(sFinalUrl, sUrl, sizeof(sFinalUrl) - 1); - bFromCache = true; - } - else{ - /* No copy, or one older than the session trusts. With validators - * from the old copy the request is conditional and a 304 renews it - * for no body bytes. If the server cannot be reached at all the old - * copy still beats nothing: a warm cache keeps working offline. The - * 20 MB limit is a sanity check, no catalog node comes near it. */ - char sReqHdrs[1024] = {'\0'}; - bool bHaveOld = (sCache[0] != '\0') && das_isfile(sCache); - if(bHaveOld) _DasNode_readValidators(sCache, sReqHdrs, sizeof(sReqHdrs)); - - pBytesAry = das_http_readUrl_ex( - sUrl, sAgent, pMgr, &httpRes, 1024*1024*20, rConSec, sReqHdrs - ); - if(pBytesAry != NULL){ - das_url_toStr(&(httpRes.url), sFinalUrl, sizeof(sFinalUrl)); - } - else if(httpRes.nCode == 304){ - daslog_debug_v("Node %s unchanged on the server, cached copy renewed", sUrl); - pBytesAry = _DasNode_readFile(sCache); - strncpy(sFinalUrl, sUrl, sizeof(sFinalUrl) - 1); - bRenewed = true; - } - else if(bHaveOld){ - daslog_warn_v("%s, using the cached copy of %s", - httpRes.sError ? httpRes.sError : "Fetch failed", sUrl); - pBytesAry = _DasNode_readFile(sCache); - strncpy(sFinalUrl, sUrl, sizeof(sFinalUrl) - 1); - bFromCache = true; - } - else{ - daslog_warn_v("%s", httpRes.sError ? httpRes.sError : "Fetch failed"); - } - } - if(pBytesAry == NULL){ - DasHttpResp_freeFields(&httpRes); - return NULL; - } - - size_t uBytes; - const char* pBytes = (const char*) (DasAry_getBytesIn(pBytesAry, DIM0, &uBytes)); - DasNode* pNode = _DasNode_fromJson(pBytes, uBytes, sUrl, sFinalUrl, sPathUri); - if(pNode != NULL){ pNode->pMgr = pMgr; pNode->sAgent = sAgent; pNode->pCache = pCache; } - - /* Only documents that parse are worth keeping, a server error page is - * not. A renewal rewrites the same bytes, which is what moves the - * copy's age, and keeps the validators it already has. */ - if((pNode != NULL) && (sCache[0] != '\0') && !bFromCache){ - _DasNode_writeCache(sCache, pBytes, uBytes); - if(!bRenewed) _DasNode_writeValidators(sCache, &httpRes); - } - - DasHttpResp_freeFields(&httpRes); - dec_DasAry(pBytesAry); /* Done with the byte array, data in json buffer now */ - return pNode; -} - -/* ************************************************************************* */ -/* Sub Node Construction */ - -DasNode* DasNode_subNode( - DasNode* pThis, const char* sRelPath, DasCredMngr* pMgr, const char* sAgent -); - -static void _DasNode_delTree(DasNode* pThis); - - /* Helper for loadSubNode and subNode * * We have a fundamental difficulty in looking for sub nodes. Since a catalog @@ -836,7 +519,7 @@ static void _DasNode_delTree(DasNode* pThis); * @param sSubPath - The sub path *WITH THE SEPARATOR ATTACHED* * */ -static bool _checkSubMatch(DasNode* pBase, const char* sSubPathWithSep) +bool _DasNode_subMatch(DasNode* pBase, const char* sSubPathWithSep) { if(!DasNode_isCatalog(pBase)) return false; @@ -886,246 +569,20 @@ static bool _checkSubMatch(DasNode* pBase, const char* sSubPathWithSep) /* sRelPath is the desired sub path from the point of this object. If it's * NULL, that means you just wanted what you already have (pThis) so there's * no point in calling this function */ -static DasNode* _DasNode_loadSubNode_dasCat( - DasCatNode* pThis, const char* sRelPath, DasCredMngr* pMgr, - const char* sAgent -){ - if(!DasNode_isJson((DasNode*)pThis)){ - daslog_error("XML catalogs not yet supported"); - return NULL; - } - const DasJdo* pDir = (DasJdo*) pThis->pContainer; - if((pDir == NULL) || (pDir->type != das_json_type_dict)){ - daslog_error_v("Catalog container node missing for node from %s", - pThis->base.sURL); - return NULL; - } - - /* Run over all the keys and see if any of them match the front of the - * requested relative path. If so try all the URLs given for full object - * realization before giving up */ - const das_json_dict_el* pEl = NULL; - size_t uLen; - const DasJdo* pChild = NULL; - const char* sChild = NULL; - const DasJdo* pChildUrls = NULL; - const das_json_ary_el* pUrlEl; - char sSubUri[560] = {'\0'}; - const char* sUrl = NULL; - const char* sSubRelPath = NULL; - DasNode* pNode = NULL; - DasNode* pDecendent = NULL; - int nUrl; - - /* Trying it implement a tree search that will auto backout and try again - * if a URL turns out to be broken, or contains bad data, or is the wrong - * child. */ - for(pEl = DasJdo_dictFirst(pDir); pEl != NULL; pEl = pEl->next){ - sChild = pEl->name->string; - - uLen = strlen(sChild); - - /* See if this child and the requested item have the same chars up to - * the length of the child */ - if(strncmp(sChild, sRelPath, uLen) != 0) continue; - - pChild = pEl->value; /* If this never get's set, then we had no - * children with matching names */ - - /* Just stop using this catalog node if it has broken syntax */ - if((pChild == NULL )||(pChild->type != das_json_type_dict)){ - daslog_error_v("Catalog error @ %s: sub item '%s' is not a dictionary", - pThis->base.sURL, pEl->name->string); - return NULL; - } - - pChildUrls = DasJdo_get(pChild, D2FRAG_URLS); - if((pChildUrls == NULL)||(pChildUrls->type != das_json_type_ary)){ - daslog_error_v("From %s: %s element of node '%s' doesn't " - "have a URLS array", pThis->base.sURL, sChild, - pThis->sContainer); - return NULL; - } - - /* Loop over URLs for child */ - nUrl = -1; - pNode = NULL; - - /* this is a slow time out.. */ - float rConSec = DASHTTP_TO_MIN * DASHTTP_TO_MULTI; - - for(pUrlEl = DasJdo_aryFirst(pChildUrls); pUrlEl != NULL; - pUrlEl = pUrlEl->next){ - pNode = NULL; - nUrl += 1; - sUrl = DasJdo_string(pUrlEl->value); - if(sUrl == NULL){ - daslog_error_v("From %s: %s/%s/%d element is not a string", - pThis->base.sURL, D2FRAG_URLS, sChild, nUrl); - continue; - } - - /* tell sub-item it's name */ - snprintf(sSubUri, 559, "%s%s%s", pThis->base.sPath, pThis->sPathSep, sChild); - - /* TODO: Split mkNode and server contact into two separate functions - * so that we know if we can try again with a slow server, of if - * we just received bad data and shouldn't waste time reading it - * again */ - pNode = _DasNode_mkNode(sUrl, sSubUri, pMgr, sAgent, rConSec, pThis->base.pCache); - if(pNode == NULL) continue; - - /* If after removing the child portion from the relative path there - * is nothing left we're done. */ - sSubRelPath = sRelPath + uLen; - - if(sSubRelPath[0] == '\0'){ - DasAry_append(pThis->pSubNodes, (const ubyte*) &pNode, 1); - DasAry_append(pThis->pSubPaths, (const ubyte*) sChild, strlen(sChild)+1); - DasAry_markEnd(pThis->pSubPaths, DIM1); - return pNode; - } - - /* There's more, see if it's possible for this child to complete the - * path. */ - if(_checkSubMatch(pNode, sSubRelPath)){ - - pDecendent = DasNode_subNode((DasNode*)pNode, sSubRelPath, pMgr, sAgent); - if(pDecendent){ - /* Worked okay, cache the child node, but return the descendant - * node (however far down it came from */ - DasAry_append(pThis->pSubNodes, (const ubyte*) &pNode, 1); - DasAry_append(pThis->pSubPaths, (const ubyte*) sChild, strlen(sChild)+1); - DasAry_markEnd(pThis->pSubPaths, DIM1); - - return pDecendent; - } - } - else{ - /* This child can't match, stop checking this URL set*/ - _DasNode_delTree(pNode); - pNode = NULL; - } - } - - /* Must have been a look-alike child (or some other failure), try again */ - _DasNode_delTree(pNode); - pNode = NULL; - } - - - /* A miss at one level is normal while the walk backs out and tries other - * URLs or children, so it is not an error here. The caller knows whether - * the whole path failed and has the user's wording of it. */ - const char* sUri = DasNode_pathUri((DasNode*)pThis); - daslog_info_v( - "Node %s (URI '%s') has no child node that starts with %s", - DasNode_label((DasNode*)pThis), sUri ? sUri : "", sRelPath - ); - - return NULL; -} - -static DasNode* _DasNode_loadSubNode_spdfCat( - DasCatNode* pThis, const char* sRelPath, DasCredMngr* pMgr, const char* sAgent -){ - daslog_error("SPDF catalogs are not yet supported"); - return NULL; -} - -static DasNode* _DasNode_loadSubNode_spaseCat( - DasCatNode* pThis, const char* sRelPath, DasCredMngr* pMgr, const char* sAgent -){ - daslog_error("Spase catalogs are not yet supported"); - return NULL; -} - -DasNode* DasNode_subNode( - DasNode* pThis, const char* sRelPath, DasCredMngr* pMgr, const char* sAgent +DasNode* new_DasNode_json( + const char* pBytes, size_t uBytes, const char* sUrl, const char* sPathUri ){ - /* Credentials and agent flow down from the root unless overridden here */ - if(pMgr == NULL) pMgr = pThis->pMgr; - if(sAgent == NULL) sAgent = pThis->sAgent; - - if((sRelPath == NULL) || (sRelPath[0] == '\0')){ - das_error(DASERR_NODE, "Empty relative path, can't lookup scheme definition"); - return NULL; - } - if(!DasNode_isCatalog(pThis)){ - daslog_error_v("Node %s from %s is a terminating node", DasNode_label(pThis), - DasNode_srcUrl(pThis)); - return NULL; - } - DasCatNode* pCat = (DasCatNode*)pThis; - - if((pCat->sPathSep[0] != '\0')&& - (strncmp(sRelPath, pCat->sPathSep, strlen(pCat->sPathSep)) == 0)){ - sRelPath += strlen(pCat->sPathSep); - } - - /* Search the local child cache first before calling on the inter-tubes */ - const char* sSubPath = NULL; - const char* sSubRelPath = NULL; - ptrdiff_t uChildren = DasAry_lengthIn(pCat->pSubNodes, DIM0); - size_t uLen = 0; - DasNode* pNode = NULL; - DasNode* pRet = NULL; - for(ptrdiff_t u = 0; u < uChildren; ++u){ - sSubPath = (const char*)DasAry_getBytesIn(pCat->pSubPaths, DIM1_AT(u), &uLen); - - /* From the DasAry's point of view, the item length includes the null - * because it has to store the null. But from strncmp's point of view - * we don't count the null because the rel path might not be a terminating - * path (i.e. it may have components below the child */ - if(uLen == 0){ - daslog_warn_v("Node at %s has a zero-length sub-path", pThis->sURL); - continue; - } - else{ - uLen -= 1; - } - - if(strncmp(sSubPath, sRelPath, uLen) == 0){ - - /* This child completes at least part of the subpath */ - pNode = *((DasNode**) DasAry_getAt(pCat->pSubNodes, vtUnknown, IDX0(u))); - - sSubRelPath = sRelPath + uLen; - if(sSubRelPath[0] == '\0'){ - /* actually it completes all of it*/ - return pNode; - } - else{ - /* There's more, see if it's possible for this child to complete - * the path, if not don't fail, just keep looking */ - if( ! _checkSubMatch(pNode, sSubRelPath)) continue; - - pRet = DasNode_subNode(pNode, sSubRelPath, pMgr, sAgent); - if(pRet != NULL) return pRet; - } - } - } - - /* I don't have this sub node: Internet! To the rescue! */ - - switch(pThis->nType){ - case d2node_collection: - case d2node_catalog: - case d2node_srcset: - return _DasNode_loadSubNode_dasCat(pCat, sRelPath, pMgr, sAgent); - case d2node_spdf_cat: - return _DasNode_loadSubNode_spdfCat(pCat, sRelPath, pMgr, sAgent); - case d2node_spase_cat: - return _DasNode_loadSubNode_spaseCat(pCat, sRelPath, pMgr, sAgent); - break; - default: - /* Should have been ruled out by constructor, but double check since - * new types may be added */ - das_error(DASERR_ASSERT, "Logic error in libdas2"); return NULL; - } + /* Every node has a URI; one built from bytes with no name is named by + where the bytes came from, or by a stand in when even that is unknown */ + char sUri[600]; + if(sUrl == NULL) sUrl = "memory:node"; + if((sPathUri == NULL)||(sPathUri[0] == '\0')) _das_uri_local(sUrl, sUri, sizeof(sUri)); + else _das_uri_expand(sPathUri, sUri, sizeof(sUri)); - return NULL; + DasNode* pNode = _DasNode_fromJson(pBytes, uBytes, sUrl, sUrl, sUri); + if(pNode) pNode->bOwner = true; + return pNode; } /* ************************************************************************* */ @@ -1135,7 +592,7 @@ DasNode* DasNode_subNode( to a caller: those are freed only by del_DasNode(). Not public: a child lives in its parent's pSubNodes cache, freeing one alone would leave a dangling entry there. */ -static void _DasNode_delTree(DasNode* pThis) +void _DasNode_delTree(DasNode* pThis) { if(pThis == NULL) return; if(pThis->bOwner) return; @@ -1227,30 +684,6 @@ bool _das_uri_isLocal(const char* sUri) return (sUri != NULL) && (strncmp(sUri, D2URI_LOCAL, sizeof(D2URI_LOCAL) - 1) == 0); } -/* ************************************************************************* */ -/* Construction */ - -DasNode* _DasNode_newUrl( - const char* sUrl, const char* sPathUri, DasCredMngr* pMgr, const char* sAgent, - const DasNodeCache* pCache -){ - /* Every node has a URI; one loaded on its own is named by where it is */ - char sUri[600]; - if((sPathUri == NULL)||(sPathUri[0] == '\0')) _das_uri_local(sUrl, sUri, sizeof(sUri)); - else _das_uri_expand(sPathUri, sUri, sizeof(sUri)); - - /* Since they want a specific URL, go for the max timeout */ - DasNode* pNode = _DasNode_mkNode(sUrl, sUri, pMgr, sAgent, DASHTTP_TO_MAX, pCache); - if(pNode) pNode->bOwner = true; - return pNode; -} - -DasNode* new_DasNode_url( - const char* sUrl, const char* sPathUri, DasCredMngr* pMgr, const char* sAgent -){ - return _DasNode_newUrl(sUrl, sPathUri, pMgr, sAgent, NULL); -} - /* Append a JSON string literal, escaping the two characters that matter */ static size_t _jsonStr(char* sOut, size_t uOut, const char* sIn) { @@ -1359,74 +792,3 @@ DasNode* _DasNode_newMountRoot( return pRoot; } -DasNode* _DasNode_new( - const char* sPathUri, DasCredMngr* pMgr, const char* sAgent, - const DasNodeCache* pCache -){ - /* Find the node using the path, start at the root */ - DasNode* pTop = NULL; - DasNode* pNode = NULL; - - /* Expanding shorthands here, in the library, means every client gets - the same spelling. Names under local: belong to nodes loaded by - location in some session; the federated roots have never heard of - them and asking would only cost a round trip. */ - char sFullUri[600] = {'\0'}; - if(sPathUri != NULL){ - sPathUri = _das_uri_expand(sPathUri, sFullUri, sizeof(sFullUri)); - if(_das_uri_isLocal(sPathUri)){ - daslog_debug_v("%s is a local: name, those are not routed through the federated catalog", sPathUri); - return NULL; - } - } - - /* Changing node lookup and time outs. New strategy, try to contact a node - if it doesn't work within the time out, try the next one. If all fail - then increase the time out and try again. - */ - - float rConSec = DASHTTP_TO_MIN; - - while((rConSec <= DASHTTP_TO_MAX )&&(pNode == NULL)){ - for(int i = 0; i < g_nDasDefRoots; ++i){ - /* Credentials should never be needed for top level nodes */ - pTop = _DasNode_mkNode(g_lDasDefRoots[i], NULL, NULL, sAgent, rConSec, pCache); - if(pTop == NULL) continue; - if(pTop->nType != d2node_catalog) continue; - - if(sPathUri == NULL){ /* They may be looking for a root node */ - pNode = pTop; - break; - } - - /* Depends on the fact that top node is a das catalog */ - pNode = _DasNode_loadSubNode_dasCat((DasCatNode*)pTop, sPathUri, pMgr, sAgent); - if(pNode != NULL) break; - } - rConSec *= DASHTTP_TO_MULTI; - } - - if(pNode == NULL){ - daslog_error_v("Failed to load the requested node %s from any of the " - "built in catalog trees", sPathUri); - return NULL; - } - - /* The caller gets this node, so mark it theirs before freeing the walk - * above it; _DasNode_delTree() then leaves it alone even when - * pTop == pNode */ - pNode->bOwner = true; - pNode->pMgr = pMgr; /* pTop was fetched without them, the kept node - hands them to whatever it loads later */ - pNode->sAgent = sAgent; - pNode->pCache = pCache; - _DasNode_delTree(pTop); - - return pNode; -} - -DasNode* new_DasNode(const char* sPathUri, DasCredMngr* pMgr, const char* sAgent) -{ - return _DasNode_new(sPathUri, pMgr, sAgent, NULL); -} - diff --git a/das3/node.h b/das3/node.h index 52dd590..268423e 100644 --- a/das3/node.h +++ b/das3/node.h @@ -291,6 +291,28 @@ DAS_API DasNode* new_DasNode( DAS_API DasNode* new_DasNode_url(const char* sUrl, const char* sPathUri, DasCredMngr* pMgr, const char* sAgent); +/** Build a node from the JSON text of a node document + * + * For programs that fetch documents themselves, a browser build in + * particular. Nothing is fetched and no file is read. The result is a + * tree top the caller owns, with no parent in memory; a container built + * this way lists its children, see DasNode_subCount(), and names the + * URLs each would load from, see DasNode_subStub(). + * + * @param pBytes The document text, need not be NUL terminated. + * @param uBytes The number of bytes in pBytes. + * @param sUrl Where the text came from, recorded as the node's source + * URL, see DasNode_srcUrl(). May be NULL. + * @param sPathUri The catalog URI of the node, shorthands expanded. NULL + * or empty names the node after sUrl under D2URI_LOCAL. + * @return A new node, to be freed with del_DasNode(), or NULL after + * logging a parse problem or an unrecognized node type. + * @memberof DasNode + */ +DAS_API DasNode* new_DasNode_json( + const char* pBytes, size_t uBytes, const char* sUrl, const char* sPathUri +); + /** Get a das2 catalog node contained item. * * This function will search down the catalog hierarchy and find the requested diff --git a/das3/node_priv.h b/das3/node_priv.h index 1ea19d1..35507e8 100644 --- a/das3/node_priv.h +++ b/das3/node_priv.h @@ -39,9 +39,10 @@ * The type is checked before casting, see DasNode_isCatalog(). Terminal * nodes (stream sources, file aggregations) are a bare struct das_node. * - * Loading. Every node begins as a JSON document that _DasNode_mkNode() - * reads from a file, from the cache, or over HTTP, and _DasNode_fromJson() - * turns into a struct of the right type. The parsed DasJdo is kept in pDom + * Loading. Every node begins as a JSON document that _DasNode_fromJson() + * turns into a struct of the right type. Where the bytes come from is + * not node.c's business: catalog.c reads files, the cache and HTTP, a + * browser build hands them to new_DasNode_json(). The parsed DasJdo is kept in pDom * for the life of the node and the accessors read from it rather than * copying fields out. DasNode_subNode() matches a relative path against * the names in the parent's container dictionary, loads the matching @@ -87,6 +88,47 @@ struct das_node { const DasNodeCache* pCache; /* borrowed, inherited by children */ }; +/* Catalog nodes have extra arrays to cache nodes */ +#define _D2C_SEP_SZ 8 +typedef struct das_catalog { + DasNode base; + + DasAry* pSubNodes; + + DasAry* pSubPaths; + + /* The json object or xml element tag which contains the sub elements */ + void* pContainer; + + /* The path separator, defaults to '/' unless overridden by the object */ + char sPathSep[_D2C_SEP_SZ]; + + /* The name of the element which contains sub elements */ + char sContainer[64]; + +} DasCatNode; + + + +/* Build a node from JSON text, see new_DasNode_json() for the public + * form. sUrl is where the text came from and sFinalUrl the same after + * redirects, both for messages. sPathUri may be NULL. The node is not + * marked as owned, the caller decides that. */ +DasNode* _DasNode_fromJson( + const char* pBytes, size_t uBytes, const char* sUrl, const char* sFinalUrl, + const char* sPathUri +); + +/* True if some child listed by pBase could be the next step of sSubPath, + * which starts with the separator pBase uses. Nothing is loaded. */ +bool _DasNode_subMatch(DasNode* pBase, const char* sSubPathWithSep); + +/* Frees a node and everything below it, except nodes a constructor handed + * to a caller: those are freed only by del_DasNode(). A child lives in + * its parent's pSubNodes cache, freeing one alone would leave a dangling + * entry there. */ +void _DasNode_delTree(DasNode* pThis); + /* Catalog URI spellings. * * Names are composed top down: the das2 root node has no separator, so From de0a67181f6ccc815b4834c038c0e1ea1db8934d Mon Sep 17 00:00:00 2001 From: cpiker Date: Wed, 28 Oct 2026 03:56:41 -0500 Subject: [PATCH 12/18] catalog: reworked for tansport fee operation * All I/O code removed from DasNode to DasCat * New state machine class provided to run URI resolution using only DasNode objects, useful for emscripten builds * Vocabulary change: root->seed. Root now is means a seed node with no URI, that's it. See doxygen docs * das_fetch command line changes. See help text. * das_fetch can mount a local catalog on a local catalog * bug fixs: Memory leak in DasHttpResp_freeFields plugged Developed-with: Claude Fable 5.1 --- buildfiles/Darwin.mak | 2 + buildfiles/Emscripten.mak | 10 +- buildfiles/Linux.mak | 2 + buildfiles/Windows.mak | 2 + buildfiles/cmake/das2CTests.cmake | 1 + buildfiles/sources.mak | 2 +- das3/catalog.c | 1005 ++++++++++++----------------- das3/catalog.h | 240 ++++--- das3/http.c | 33 +- das3/http.h | 17 - das3/node.c | 556 +++++++++++++--- das3/node.h | 526 +++++++-------- das3/node_priv.h | 211 ++---- das3/source.c | 19 +- das3/source.h | 6 +- das3/util.c | 52 ++ das3/util.h | 38 ++ test/TestCat.c | 171 ++--- test/TestCatalog.c | 67 +- test/TestSource.c | 36 +- test/TestWalk.c | 413 ++++++++++++ test/das_fetch_test.sh | 80 +-- test/das_walk/a.json | 9 + test/das_walk/ab.json | 9 + test/das_walk/fedroot.json | 10 + test/das_walk/junk.json | 1 + test/das_walk/root.json | 33 + test/das_walk/src.json | 8 + utilities/das_fetch.c | 228 ++++--- 29 files changed, 2250 insertions(+), 1537 deletions(-) create mode 100644 test/TestWalk.c create mode 100644 test/das_walk/a.json create mode 100644 test/das_walk/ab.json create mode 100644 test/das_walk/fedroot.json create mode 100644 test/das_walk/junk.json create mode 100644 test/das_walk/root.json create mode 100644 test/das_walk/src.json diff --git a/buildfiles/Darwin.mak b/buildfiles/Darwin.mak index 1c6f668..92ac5fd 100644 --- a/buildfiles/Darwin.mak +++ b/buildfiles/Darwin.mak @@ -374,6 +374,8 @@ test_main: $(BD) $(BD)/$(TARG).a $(BUILD_TEST_PROGS) $(BULID_UTIL_PROGS) $(BD)/TestUri $(BD)/uri_tplt @echo "INFO: Running unit test for credentials manager, $(BD)/TestCredMngr..." @$(BD)/TestCredMngr $(BD) + @echo "INFO: Running unit test for catalog walks over local nodes, $(BD)/TestWalk..." + @$(BD)/TestWalk @echo "INFO: Running unit test for catalog sessions over local nodes, $(BD)/TestCat..." @$(BD)/TestCat @echo "INFO: Running unit test for source descriptions and requisitions, $(BD)/TestSource..." diff --git a/buildfiles/Emscripten.mak b/buildfiles/Emscripten.mak index d8422e1..f27d636 100644 --- a/buildfiles/Emscripten.mak +++ b/buildfiles/Emscripten.mak @@ -22,8 +22,10 @@ # # What is left out, and why: # -# http.c credentials.c node.c sockets and OpenSSL do not exist in a -# browser; fetch() on the JS side replaces them +# http.c credentials.c catalog.c sockets, OpenSSL and the node cache do +# not exist in a browser. fetch() on the JS +# side drives DasNodeWalk instead; node.c, +# source.c and req.c never touch a transport # dft.c FFTW is not built for wasm yet. DFT=yes # puts it back given FFTW_INC. # cli.c send.c program scaffolding, nothing a library @@ -50,7 +52,7 @@ TARG=libdas3 include buildfiles/sources.mak -NET_SRCS:=das3/http.c das3/credentials.c das3/node.c +NET_SRCS:=das3/http.c das3/credentials.c das3/catalog.c PROG_SRCS:=das3/cli.c das3/send.c DFT_SRCS:=das3/dft.c @@ -81,7 +83,7 @@ HDRS:=$(notdir $(LIB_HDRS)) # opens a URL, reads credentials, or needs SPICE stays out. TEST_PROGS:=TestValue TestDatum TestGen TestForm TestCplx TestVar \ TestVarSubset TestDim TestDesc TestUnits TestTT2000 TestArray TestDs TestIter \ - TestRaggedEncode TestFilter TestUri TestV3Read TestFeed + TestRaggedEncode TestFilter TestUri TestV3Read TestFeed TestWalk ############################################################################## # Build definitions diff --git a/buildfiles/Linux.mak b/buildfiles/Linux.mak index a02f695..6e443df 100644 --- a/buildfiles/Linux.mak +++ b/buildfiles/Linux.mak @@ -372,6 +372,8 @@ test_main: $(BD) $(BD)/$(TARG).a $(BUILD_TEST_PROGS) $(BUILD_UTIL_PROGS) $(BD)/TestUri $(BD)/uri_tplt @echo "INFO: Running unit test for credentials manager, $(BD)/TestCredMngr..." @$(BD)/TestCredMngr $(BD) + @echo "INFO: Running unit test for catalog walks over local nodes, $(BD)/TestWalk..." + @$(BD)/TestWalk @echo "INFO: Running unit test for catalog sessions over local nodes, $(BD)/TestCat..." @$(BD)/TestCat @echo "INFO: Running unit test for source descriptions and requisitions, $(BD)/TestSource..." diff --git a/buildfiles/Windows.mak b/buildfiles/Windows.mak index d8e067a..d2d7fa4 100644 --- a/buildfiles/Windows.mak +++ b/buildfiles/Windows.mak @@ -388,6 +388,8 @@ test_main: $(BD)\TestUri.exe $(BD)\uri_tplt @echo INFO: Running unit test for credentials manager, $(BD)\TestCredMngr... @$(BD)\TestCredMngr.exe $(BD) + @echo INFO: Running unit test for catalog walks over local nodes, $(BD)\TestWalk... + @$(BD)\TestWalk.exe @echo INFO: Running unit test for catalog sessions over local nodes, $(BD)\TestCat... @$(BD)\TestCat.exe @echo INFO: Running unit test for source descriptions and requisitions, $(BD)\TestSource... diff --git a/buildfiles/cmake/das2CTests.cmake b/buildfiles/cmake/das2CTests.cmake index fca8f2f..2f805f2 100644 --- a/buildfiles/cmake/das2CTests.cmake +++ b/buildfiles/cmake/das2CTests.cmake @@ -64,6 +64,7 @@ _das_test(TestRaggedEncode main TestRaggedEncode _das_test(TestFilter main TestFilter) _das_test(TestUri main TestUri "${_bd}/uri_tplt") _das_test(TestCredMngr main TestCredMngr "${_bd}") +_das_test(TestWalk main TestWalk) _das_test(TestCat main TestCat) _das_test(TestSource main TestSource) _das_test(TestHttp main TestHttp) diff --git a/buildfiles/sources.mak b/buildfiles/sources.mak index 440d19c..f9a3178 100644 --- a/buildfiles/sources.mak +++ b/buildfiles/sources.mak @@ -61,7 +61,7 @@ CDF_UTIL_SRCS = utilities/das3_cdf.c utilities/das3_from_cdf.c \ utilities/das2_from_cdf.c TEST_SRCS = test/TestUnits.c test/TestArray.c test/TestDs.c test/TestBuilder.c \ - test/TestAuth.c test/TestCatalog.c test/TestCat.c test/TestSource.c test/TestHttp.c test/TestTT2000.c test/ex_das_cli.c \ + test/TestAuth.c test/TestCatalog.c test/TestWalk.c test/TestCat.c test/TestSource.c test/TestHttp.c test/TestTT2000.c test/ex_das_cli.c \ test/ex_das_ephem.c test/TestCredMngr.c test/TestV3Read.c test/TestFeed.c \ test/TestProp.c \ test/TestIter.c test/TestUri.c test/TestFilter.c test/TestValue.c \ diff --git a/das3/catalog.c b/das3/catalog.c index 2e406ff..87fa1c9 100644 --- a/das3/catalog.c +++ b/das3/catalog.c @@ -18,10 +18,11 @@ */ /* A catalog session. Grew up as a pilot inside utilities/das_fetch.c and - * moved here once a second client wanted the same root policy. + * moved here once a second client wanted the same rules for where paths + * start. * Policy misuse is reported with daslog_error and DASERR_CAT rather than * das_error, because the usual source of a bad policy is a command line - * argument and a program should get to decide its own exit code for that. */ + * argument and a program should get to decide its exit code for that. */ #define _POSIX_C_SOURCE 200112L @@ -41,6 +42,14 @@ #include "source.h" #include "catalog.h" +/* Where this session keeps copies of nodes fetched over HTTP and how long + * it trusts them. Empty sDir means no caching; nMaxAgeSec of zero or less + * means always ask the server, see _dascat_fetch(). */ +typedef struct das_node_cache { + char sDir[256]; + int nMaxAgeSec; +} DasNodeCache; + struct das_cat_mount { char* sUrl; char* sUri; @@ -51,41 +60,54 @@ struct das_cat { char* sAgent; /* owned copy, or NULL for the library default */ DasNodeCache* pCache; /* owned, NULL until DasCat_setCache() */ - /* Root policy. Relative paths start at sPrefix in the federated - * catalog, or with mounts at the root node if one was set, else at - * the first mounted node, or at the prefix if one was given. - * - * A root is a mount that turns off fall through: the mounts are then - * the whole catalog. Keeping one list means a root and further mounts - * mix freely, which is how an offline test catalog is assembled from a - * handful of files. */ + /* Where paths start. Relative paths get sPrefix in front, or with a + * seed set, the seed's prefix and separator; the two are one knob + * with two setters, and only one may be used. Mounts are consulted + * before the federated catalog for any URI they cover, opportunistic + * grafts rather than a policy. A seed is a mount that turns the + * federated catalog off: the mounts are then the whole catalog, which + * is how an offline catalog is assembled from a handful of files. */ char* sPrefix; /* never NULL */ - bool bPrefixGiven; + bool bPrefixGiven; /* DasCat_setPrefix() was called */ struct das_cat_mount* aMounts; /* named nodes loaded by location */ size_t uMounts; - bool bIsolated; /* set by DasCat_setRoot(): no fall through */ - size_t uRoot; /* index of the root in aMounts when isolated */ - - DasNode* pRoot; /* the mount root, built on first use */ - DasNode** aRoots; /* every node a constructor gave us, freed with the session */ - size_t uRoots; - size_t uRootCap; + bool bSeed; /* set by DasCat_setSeed(): the seed and the mounts are the whole catalog */ + char* sSeedUrl; /* owned, where the seed loads from */ + char* sSeedUri; /* owned, its prefix, "" for a root */ + DasNode* pSeed; /* the seed node, loaded on first use */ + + DasNode* pTable; /* the mount table, built on first use */ + DasNode* pDas2; /* the compiled in seed, loaded on first use; the + tree below it is the session's memory of the + federated catalog */ + DasNode** aOwned; /* every tree top the session holds, freed with it */ + size_t uOwned; + size_t uOwnedCap; char* sLastPath; /* the caller's path to the last resolve, or NULL */ char* sLastUri; /* that path as the policy understood it */ + char* sNear; /* after a failed walk, the deepest container it searched */ + char* sNearRest; /* and the path below it, see DasCat_nearest() */ + DasNode* pNearNode; /* that container when it is a root and so has no URI to + resolve again: the walk's start node, borrowed */ }; +/* Defined further down, used throughout */ +static char* _strdupz(const char* s); +static DasNode* _dascat_loadUrl( + const char* sUrl, const char* sPathUri, DasCredMngr* pMgr, const char* sAgent, + const DasNodeCache* pCache +); + /* ************************************************************************* */ /* Loading nodes: the native driver * - * Everything below, down to the small helpers, is what node.c does not do: - * read a file, consult the node cache, make an HTTP request, and walk a - * catalog tree by loading one node after another. node.c only parses - * bytes it is handed and reads the result. A client with its own - * transport, a browser build in particular, uses node.h alone and never - * links this file. Moved here from node.c as one piece; the recursive - * walk is replaced by DasNodeWalk in the next step. Until then the - * public pull constructors in node.h are implemented here. */ + * node.c doesn't read files or open sockets. It parses the documents it + * is handed and, through DasNodeWalk, says which document it needs next. + * What follows is the driver for programs on a real machine: it reads + * files, keeps the node cache, makes the HTTP requests, and loops a walk + * until it stops. A client that fetches for itself, a browser build in + * particular, just use node.h alone and exclude this file. */ /* ************************************************************************* */ /* Local file nodes. @@ -97,7 +119,7 @@ struct das_cat { * Only the path forms "file:///abs/path" and "file://rel/path" are handled, * host names in file URLs are not. */ -static bool _DasNode_isFileUrl(const char* sUrl) +static bool _dascat_isFileUrl(const char* sUrl) { if(sUrl == NULL) return false; if(strncmp(sUrl, "http://", 7) == 0) return false; @@ -105,7 +127,7 @@ static bool _DasNode_isFileUrl(const char* sUrl) return true; } -static DasAry* _DasNode_readFile(const char* sUrl) +static DasAry* _dascat_readFile(const char* sUrl) { const char* sPath = sUrl; if(strncmp(sUrl, "file://", 7) == 0) sPath += 7; @@ -139,16 +161,16 @@ static DasAry* _DasNode_readFile(const char* sUrl) /* ************************************************************************* */ /* Node cache */ -/* Only a DasCat has cache settings, nodes made through the public - * constructors in node.h carry NULL and are never cached. One knob, per - * session, instead of a second process wide one to confuse it with. */ +/* Only a DasCat has cache settings, nodes made through the sessionless + * constructors in node.h aren't cached. One knob, per session, instead + * of a second process wide one to confuse it with. */ /* Cache file name for a URL: the URL minus its scheme with everything * outside [A-Za-z0-9.-] turned into '_', then a hash of the whole URL so * that two URLs which sanitize alike still get separate files. Readable * names make an 'ls' of the cache directory useful when debugging a * catalog, a bare hash would not. */ -static bool _DasNode_cachePath( +static bool _dascat_cachePath( const DasNodeCache* pCache, const char* sUrl, char* sPath, size_t uLen ){ if((pCache == NULL)||(pCache->sDir[0] == '\0')) return false; @@ -176,19 +198,19 @@ static bool _DasNode_cachePath( } /* A fresh enough cached copy, or NULL */ -static DasAry* _DasNode_readCache(const DasNodeCache* pCache, const char* sPath) +static DasAry* _dascat_readCache(const DasNodeCache* pCache, const char* sPath) { if((pCache == NULL)||(pCache->nMaxAgeSec <= 0)) return NULL; struct stat st; if(stat(sPath, &st) != 0) return NULL; if((time(NULL) - st.st_mtime) > pCache->nMaxAgeSec) return NULL; - return _DasNode_readFile(sPath); + return _dascat_readFile(sPath); } /* Written to a side file then renamed, so a reader never sees a partial * document when two programs walk the catalog at once. Failures only cost * the cache, so they are logged at debug level and otherwise ignored. */ -static void _DasNode_writeCache(const char* sPath, const char* pBytes, size_t uBytes) +static void _dascat_writeCache(const char* sPath, const char* pBytes, size_t uBytes) { char sTmp[600]; if(snprintf(sTmp, sizeof(sTmp), "%s.%ld.tmp", sPath, (long)time(NULL)) >= (int)sizeof(sTmp)) return; @@ -216,19 +238,19 @@ static void _DasNode_writeCache(const char* sPath, const char* pBytes, size_t uB * as If-None-Match and If-Modified-Since, and a 304 renews the copy for * one round trip and no body. Both are kept because servers differ: the * das2.org tree sends both, GitHub raw sends ETag only, dasflex today - * sends neither. The date is never derived from the cache file's own + * sends neither. The date is never derived from the cache file's * mtime, which renewal moves forward. */ -static void _DasNode_hdrPath(const char* sCache, char* sHdr, size_t uLen) +static void _dascat_hdrPath(const char* sCache, char* sHdr, size_t uLen) { snprintf(sHdr, uLen, "%s.hdr", sCache); } /* Request header lines for the cached copy at sCache, "" if it has none */ -static void _DasNode_readValidators(const char* sCache, char* sReq, size_t uLen) +static void _dascat_readValidators(const char* sCache, char* sReq, size_t uLen) { sReq[0] = '\0'; char sHdr[600]; - _DasNode_hdrPath(sCache, sHdr, sizeof(sHdr)); + _dascat_hdrPath(sCache, sHdr, sizeof(sHdr)); FILE* pFile = fopen(sHdr, "r"); if(pFile == NULL) return; @@ -252,10 +274,10 @@ static void _DasNode_readValidators(const char* sCache, char* sReq, size_t uLen) /* Save the validators a 200 carried, or drop the side file if it carried * none, so a validator from an older body is never sent for a newer one */ -static void _DasNode_writeValidators(const char* sCache, const DasHttpResp* pRes) +static void _dascat_writeValidators(const char* sCache, const DasHttpResp* pRes) { char sHdr[600]; - _DasNode_hdrPath(sCache, sHdr, sizeof(sHdr)); + _dascat_hdrPath(sCache, sHdr, sizeof(sHdr)); char sETag[256] = {'\0'}; char sMod[128] = {'\0'}; bool bETag = DasHttpResp_header(pRes, "ETag", sETag, sizeof(sETag)); @@ -272,444 +294,232 @@ static void _DasNode_writeValidators(const char* sCache, const DasHttpResp* pRes fclose(pFile); } -/* make a node from a *specific* URL, does not call das_error functions unless - * a memory allocation error occurs because network operations are assumed to - * fail sometimes. Returns a new node allocated on the heap if the URL - * contained a recognizable DasCatalog object - */ -static DasNode* _DasNode_mkNode( - const char* sUrl, const char* sPathUri, DasCredMngr* pMgr, - const char* sAgent, float rConSec, const DasNodeCache* pCache +/* ************************************************************************* */ +/* One document, from wherever it lives */ + +struct dascat_fetch { + DasAry* pBytes; /* the document, NULL when nothing came back */ + char sFinal[512]; /* the URL after redirects, for messages */ + char sCache[512]; /* where a copy goes, "" when not cached */ + bool bFromCache; /* copy used as is, nothing to write back */ + bool bRenewed; /* server confirmed the copy, rewrite it to move its age */ + bool bNoAnswer; /* no HTTP status came back: refused, timed out, no route */ + DasHttpResp res; +}; + +/* Calls das_error only when memory runs out, network operations are + * assumed to fail sometimes. The 20 MB limit is a sanity check, no + * catalog node comes near it. */ +static void _dascat_fetch( + DasCredMngr* pMgr, const char* sAgent, const DasNodeCache* pCache, + const char* sUrl, float rConSec, struct dascat_fetch* pF ){ - /* Save the final URL, may be needed for error messages, may not match - * sUrl due to re-directs */ - char sFinalUrl[512] = {'\0'}; - - DasAry* pBytesAry = NULL; - char sCache[512] = {'\0'}; - bool bFromCache = false; /* copy used as is, nothing to write back */ - bool bRenewed = false; /* server confirmed the copy, rewrite it to move its age */ - DasHttpResp httpRes; - memset(&httpRes, 0, sizeof(DasHttpResp)); - - if(_DasNode_isFileUrl(sUrl)){ - pBytesAry = _DasNode_readFile(sUrl); - strncpy(sFinalUrl, sUrl, sizeof(sFinalUrl) - 1); + memset(pF, 0, sizeof(*pF)); + pF->res.nCode = -1; + + if(_dascat_isFileUrl(sUrl)){ + pF->pBytes = _dascat_readFile(sUrl); + strncpy(pF->sFinal, sUrl, sizeof(pF->sFinal) - 1); + return; } - else if(_DasNode_cachePath(pCache, sUrl, sCache, sizeof(sCache)) && - ((pBytesAry = _DasNode_readCache(pCache, sCache)) != NULL)){ + if(_dascat_cachePath(pCache, sUrl, pF->sCache, sizeof(pF->sCache)) && + ((pF->pBytes = _dascat_readCache(pCache, pF->sCache)) != NULL)){ daslog_debug_v("Node %s read from cache", sUrl); - strncpy(sFinalUrl, sUrl, sizeof(sFinalUrl) - 1); - bFromCache = true; - } - else{ - /* No copy, or one older than the session trusts. With validators - * from the old copy the request is conditional and a 304 renews it - * for no body bytes. If the server cannot be reached at all the old - * copy still beats nothing: a warm cache keeps working offline. The - * 20 MB limit is a sanity check, no catalog node comes near it. */ - char sReqHdrs[1024] = {'\0'}; - bool bHaveOld = (sCache[0] != '\0') && das_isfile(sCache); - if(bHaveOld) _DasNode_readValidators(sCache, sReqHdrs, sizeof(sReqHdrs)); - - pBytesAry = das_http_readUrl_ex( - sUrl, sAgent, pMgr, &httpRes, 1024*1024*20, rConSec, sReqHdrs - ); - if(pBytesAry != NULL){ - das_url_toStr(&(httpRes.url), sFinalUrl, sizeof(sFinalUrl)); - } - else if(httpRes.nCode == 304){ - daslog_debug_v("Node %s unchanged on the server, cached copy renewed", sUrl); - pBytesAry = _DasNode_readFile(sCache); - strncpy(sFinalUrl, sUrl, sizeof(sFinalUrl) - 1); - bRenewed = true; - } - else if(bHaveOld){ - daslog_warn_v("%s, using the cached copy of %s", - httpRes.sError ? httpRes.sError : "Fetch failed", sUrl); - pBytesAry = _DasNode_readFile(sCache); - strncpy(sFinalUrl, sUrl, sizeof(sFinalUrl) - 1); - bFromCache = true; - } - else{ - daslog_warn_v("%s", httpRes.sError ? httpRes.sError : "Fetch failed"); - } - } - if(pBytesAry == NULL){ - DasHttpResp_freeFields(&httpRes); - return NULL; - } - - size_t uBytes; - const char* pBytes = (const char*) (DasAry_getBytesIn(pBytesAry, DIM0, &uBytes)); - DasNode* pNode = _DasNode_fromJson(pBytes, uBytes, sUrl, sFinalUrl, sPathUri); - if(pNode != NULL){ pNode->pMgr = pMgr; pNode->sAgent = sAgent; pNode->pCache = pCache; } - - /* Only documents that parse are worth keeping, a server error page is - * not. A renewal rewrites the same bytes, which is what moves the - * copy's age, and keeps the validators it already has. */ - if((pNode != NULL) && (sCache[0] != '\0') && !bFromCache){ - _DasNode_writeCache(sCache, pBytes, uBytes); - if(!bRenewed) _DasNode_writeValidators(sCache, &httpRes); + strncpy(pF->sFinal, sUrl, sizeof(pF->sFinal) - 1); + pF->bFromCache = true; + return; } - DasHttpResp_freeFields(&httpRes); - dec_DasAry(pBytesAry); /* Done with the byte array, data in json buffer now */ - return pNode; -} - -/* ************************************************************************* */ -/* Sub Node Construction */ - + /* No copy, or one older than the session trusts. With validators from + * the old copy the request is conditional and a 304 renews it for no + * body bytes. If the server cannot be reached at all the old copy + * still beats nothing: a warm cache keeps working offline. */ + char sReqHdrs[1024] = {'\0'}; + bool bHaveOld = (pF->sCache[0] != '\0') && das_isfile(pF->sCache); + if(bHaveOld) _dascat_readValidators(pF->sCache, sReqHdrs, sizeof(sReqHdrs)); -static DasNode* _DasNode_loadSubNode_dasCat( - DasCatNode* pThis, const char* sRelPath, DasCredMngr* pMgr, - const char* sAgent -){ - if(!DasNode_isJson((DasNode*)pThis)){ - daslog_error("XML catalogs not yet supported"); - return NULL; + pF->pBytes = das_http_readUrl_ex( + sUrl, sAgent, pMgr, &pF->res, 1024*1024*20, rConSec, sReqHdrs + ); + if(pF->pBytes != NULL){ + das_url_toStr(&(pF->res.url), pF->sFinal, sizeof(pF->sFinal)); + return; } - const DasJdo* pDir = (DasJdo*) pThis->pContainer; - - if((pDir == NULL) || (pDir->type != das_json_type_dict)){ - daslog_error_v("Catalog container node missing for node from %s", - pThis->base.sURL); - return NULL; + strncpy(pF->sFinal, sUrl, sizeof(pF->sFinal) - 1); + if(pF->res.nCode == 304){ + daslog_debug_v("Node %s unchanged on the server, cached copy renewed", sUrl); + pF->pBytes = _dascat_readFile(pF->sCache); + pF->bRenewed = true; } - - /* Run over all the keys and see if any of them match the front of the - * requested relative path. If so try all the URLs given for full object - * realization before giving up */ - const das_json_dict_el* pEl = NULL; - size_t uLen; - const DasJdo* pChild = NULL; - const char* sChild = NULL; - const DasJdo* pChildUrls = NULL; - const das_json_ary_el* pUrlEl; - char sSubUri[560] = {'\0'}; - const char* sUrl = NULL; - const char* sSubRelPath = NULL; - DasNode* pNode = NULL; - DasNode* pDecendent = NULL; - int nUrl; - - /* Trying it implement a tree search that will auto backout and try again - * if a URL turns out to be broken, or contains bad data, or is the wrong - * child. */ - for(pEl = DasJdo_dictFirst(pDir); pEl != NULL; pEl = pEl->next){ - sChild = pEl->name->string; - - uLen = strlen(sChild); - - /* See if this child and the requested item have the same chars up to - * the length of the child */ - if(strncmp(sChild, sRelPath, uLen) != 0) continue; - - pChild = pEl->value; /* If this never get's set, then we had no - * children with matching names */ - - /* Just stop using this catalog node if it has broken syntax */ - if((pChild == NULL )||(pChild->type != das_json_type_dict)){ - daslog_error_v("Catalog error @ %s: sub item '%s' is not a dictionary", - pThis->base.sURL, pEl->name->string); - return NULL; - } - - pChildUrls = DasJdo_get(pChild, D2FRAG_URLS); - if((pChildUrls == NULL)||(pChildUrls->type != das_json_type_ary)){ - daslog_error_v("From %s: %s element of node '%s' doesn't " - "have a URLS array", pThis->base.sURL, sChild, - pThis->sContainer); - return NULL; - } - - /* Loop over URLs for child */ - nUrl = -1; - pNode = NULL; - - /* this is a slow time out.. */ - float rConSec = DASHTTP_TO_MIN * DASHTTP_TO_MULTI; - - for(pUrlEl = DasJdo_aryFirst(pChildUrls); pUrlEl != NULL; - pUrlEl = pUrlEl->next){ - pNode = NULL; - nUrl += 1; - sUrl = DasJdo_string(pUrlEl->value); - if(sUrl == NULL){ - daslog_error_v("From %s: %s/%s/%d element is not a string", - pThis->base.sURL, D2FRAG_URLS, sChild, nUrl); - continue; - } - - /* tell sub-item it's name */ - snprintf(sSubUri, 559, "%s%s%s", pThis->base.sPath, pThis->sPathSep, sChild); - - /* TODO: Split mkNode and server contact into two separate functions - * so that we know if we can try again with a slow server, of if - * we just received bad data and shouldn't waste time reading it - * again */ - pNode = _DasNode_mkNode(sUrl, sSubUri, pMgr, sAgent, rConSec, pThis->base.pCache); - if(pNode == NULL) continue; - - /* If after removing the child portion from the relative path there - * is nothing left we're done. */ - sSubRelPath = sRelPath + uLen; - - if(sSubRelPath[0] == '\0'){ - DasAry_append(pThis->pSubNodes, (const ubyte*) &pNode, 1); - DasAry_append(pThis->pSubPaths, (const ubyte*) sChild, strlen(sChild)+1); - DasAry_markEnd(pThis->pSubPaths, DIM1); - return pNode; - } - - /* There's more, see if it's possible for this child to complete the - * path. */ - if(_DasNode_subMatch(pNode, sSubRelPath)){ - - pDecendent = DasNode_subNode((DasNode*)pNode, sSubRelPath, pMgr, sAgent); - if(pDecendent){ - /* Worked okay, cache the child node, but return the descendant - * node (however far down it came from */ - DasAry_append(pThis->pSubNodes, (const ubyte*) &pNode, 1); - DasAry_append(pThis->pSubPaths, (const ubyte*) sChild, strlen(sChild)+1); - DasAry_markEnd(pThis->pSubPaths, DIM1); - - return pDecendent; - } - } - else{ - /* This child can't match, stop checking this URL set*/ - _DasNode_delTree(pNode); - pNode = NULL; - } - } - - /* Must have been a look-alike child (or some other failure), try again */ - _DasNode_delTree(pNode); - pNode = NULL; + else if(bHaveOld){ + daslog_warn_v("%s, using the cached copy of %s", + pF->res.sError ? pF->res.sError : "Fetch failed", sUrl); + pF->pBytes = _dascat_readFile(pF->sCache); + pF->bFromCache = true; + } + else{ + daslog_warn_v("%s", pF->res.sError ? pF->res.sError : "Fetch failed"); + pF->bNoAnswer = (pF->res.nCode < 0); } - - - /* A miss at one level is normal while the walk backs out and tries other - * URLs or children, so it is not an error here. The caller knows whether - * the whole path failed and has the user's wording of it. */ - const char* sUri = DasNode_pathUri((DasNode*)pThis); - daslog_info_v( - "Node %s (URI '%s') has no child node that starts with %s", - DasNode_label((DasNode*)pThis), sUri ? sUri : "", sRelPath - ); - - return NULL; } -static DasNode* _DasNode_loadSubNode_spdfCat( - DasCatNode* pThis, const char* sRelPath, DasCredMngr* pMgr, const char* sAgent -){ - daslog_error("SPDF catalogs are not yet supported"); - return NULL; +/* Keep a copy of a document the walk accepted. Only documents that parse + * are worth keeping, a server error page is not. A renewal rewrites the + * same bytes, which is what moves the copy's age, and keeps the + * validators it already has. */ +static void _dascat_keep(const struct dascat_fetch* pF, const char* pBytes, size_t uBytes) +{ + if((pF->sCache[0] == '\0') || pF->bFromCache) return; + _dascat_writeCache(pF->sCache, pBytes, uBytes); + if(!pF->bRenewed) _dascat_writeValidators(pF->sCache, &pF->res); } -static DasNode* _DasNode_loadSubNode_spaseCat( - DasCatNode* pThis, const char* sRelPath, DasCredMngr* pMgr, const char* sAgent -){ - daslog_error("Spase catalogs are not yet supported"); - return NULL; +static void _dascat_fetchDone(struct dascat_fetch* pF) +{ + DasHttpResp_freeFields(&pF->res); + if(pF->pBytes != NULL) dec_DasAry(pF->pBytes); + pF->pBytes = NULL; } -DasNode* DasNode_subNode( - DasNode* pThis, const char* sRelPath, DasCredMngr* pMgr, const char* sAgent +/* ************************************************************************* */ +/* Driving a walk */ + +/* Loop a walk to its end. pbNoAnswer is set when some request got no + * answer at all, the one failure a longer timeout might cure. A path the + * catalog does not have fails through the walk the same way, and waiting + * longer would not change that. */ +static DasNode* _dascat_drive( + DasNodeWalk* pWalk, DasCredMngr* pMgr, const char* sAgent, + const DasNodeCache* pCache, float rConSec, bool* pbNoAnswer ){ - /* Credentials and agent flow down from the root unless overridden here */ - if(pMgr == NULL) pMgr = pThis->pMgr; - if(sAgent == NULL) sAgent = pThis->sAgent; - - if((sRelPath == NULL) || (sRelPath[0] == '\0')){ - das_error(DASERR_NODE, "Empty relative path, can't lookup scheme definition"); - return NULL; - } - if(!DasNode_isCatalog(pThis)){ - daslog_error_v("Node %s from %s is a terminating node", DasNode_label(pThis), - DasNode_srcUrl(pThis)); - return NULL; - } - DasCatNode* pCat = (DasCatNode*)pThis; - - if((pCat->sPathSep[0] != '\0')&& - (strncmp(sRelPath, pCat->sPathSep, strlen(pCat->sPathSep)) == 0)){ - sRelPath += strlen(pCat->sPathSep); - } - - /* Search the local child cache first before calling on the inter-tubes */ - const char* sSubPath = NULL; - const char* sSubRelPath = NULL; - ptrdiff_t uChildren = DasAry_lengthIn(pCat->pSubNodes, DIM0); - size_t uLen = 0; - DasNode* pNode = NULL; - DasNode* pRet = NULL; - for(ptrdiff_t u = 0; u < uChildren; ++u){ - sSubPath = (const char*)DasAry_getBytesIn(pCat->pSubPaths, DIM1_AT(u), &uLen); - - /* From the DasAry's point of view, the item length includes the null - * because it has to store the null. But from strncmp's point of view - * we don't count the null because the rel path might not be a terminating - * path (i.e. it may have components below the child */ - if(uLen == 0){ - daslog_warn_v("Node at %s has a zero-length sub-path", pThis->sURL); - continue; + struct dascat_fetch f; + while(DasNodeWalk_state(pWalk) == das_walk_fetch){ + const char* sUrl = DasNodeWalk_url(pWalk, 0); + _dascat_fetch(pMgr, sAgent, pCache, sUrl, rConSec, &f); + if(f.pBytes != NULL){ + size_t uLen = 0; + const char* p = (const char*)DasAry_getBytesIn(f.pBytes, DIM0, &uLen); + if(DasNodeWalk_feed(pWalk, 0, p, uLen, f.sFinal) == DAS_OKAY) + _dascat_keep(&f, p, uLen); } else{ - uLen -= 1; - } - - if(strncmp(sSubPath, sRelPath, uLen) == 0){ - - /* This child completes at least part of the subpath */ - pNode = *((DasNode**) DasAry_getAt(pCat->pSubNodes, vtUnknown, IDX0(u))); - - sSubRelPath = sRelPath + uLen; - if(sSubRelPath[0] == '\0'){ - /* actually it completes all of it*/ - return pNode; - } - else{ - /* There's more, see if it's possible for this child to complete - * the path, if not don't fail, just keep looking */ - if( ! _DasNode_subMatch(pNode, sSubRelPath)) continue; - - pRet = DasNode_subNode(pNode, sSubRelPath, pMgr, sAgent); - if(pRet != NULL) return pRet; - } + if(f.bNoAnswer && (pbNoAnswer != NULL)) *pbNoAnswer = true; + DasNodeWalk_fail(pWalk, 0, f.res.sError); } + _dascat_fetchDone(&f); } + return DasNodeWalk_result(pWalk); +} - /* I don't have this sub node: Internet! To the rescue! */ - - switch(pThis->nType){ - case d2node_collection: - case d2node_catalog: - case d2node_srcset: - return _DasNode_loadSubNode_dasCat(pCat, sRelPath, pMgr, sAgent); - case d2node_spdf_cat: - return _DasNode_loadSubNode_spdfCat(pCat, sRelPath, pMgr, sAgent); - case d2node_spase_cat: - return _DasNode_loadSubNode_spaseCat(pCat, sRelPath, pMgr, sAgent); - break; - default: - /* Should have been ruled out by constructor, but double check since - * new types may be added */ - das_error(DASERR_ASSERT, "Logic error in libdas2"); return NULL; +/* Remember what a failed walk can say, see DasCat_nearest() */ +static void _DasCat_setNear(DasCat* pThis, const DasNodeWalk* pWalk, DasNode* pFrom) +{ + free(pThis->sNear); pThis->sNear = NULL; + free(pThis->sNearRest); pThis->sNearRest = NULL; + pThis->pNearNode = NULL; + const char* sRest = NULL; + const char* sNear = DasNodeWalk_nearest(pWalk, &sRest); + if(sNear == NULL) return; + /* An empty URI is a root, which cannot be resolved again by name; it + can only be the node the walk started from. The mount table is a + root too, but nothing under it that failed is worth pointing at. */ + if(sNear[0] == '\0'){ + if(pFrom == pThis->pTable) return; + pThis->pNearNode = pFrom; } + else{ + pThis->sNear = das_strdup(sNear); + } + pThis->sNearRest = _strdupz(sRest); +} + +/* A path below a node, with this session's settings. Children get a + * slower timeout than roots since there is no second copy of the tree to + * fall back to. */ +static DasNode* _dascat_walkFrom(DasCat* pThis, DasNode* pFrom, const char* sRel) +{ + DasNodeWalk* pWalk = new_DasNodeWalk(pFrom, sRel); + if(pWalk == NULL) return NULL; + DasNode* pNode = _dascat_drive( + pWalk, pThis->pMgr, pThis->sAgent, pThis->pCache, DASHTTP_TO_MIN * DASHTTP_TO_MULTI, NULL + ); + _DasCat_setNear(pThis, pWalk, pFrom); + del_DasNodeWalk(pWalk); + return pNode; +} +/* The compiled in seed, from whichever of its mirrors answers. Each is + * tried in turn with a short connect timeout and the round is repeated + * with longer ones, but only when some server gave no answer, so one slow + * mirror does not stall the load. The seed is a tree top the caller + * owns; everything in the federated catalog is a walk from it. */ +static DasNode* _dascat_loadSeed(DasCredMngr* pMgr, const char* sAgent, const DasNodeCache* pCache) +{ + float rConSec = DASHTTP_TO_MIN; + while(rConSec <= DASHTTP_TO_MAX){ + DasNodeWalk* pWalk = new_DasNodeWalk_seed(); + if(pWalk == NULL) return NULL; + bool bNoAnswer = false; + DasNode* pRoot = _dascat_drive(pWalk, pMgr, sAgent, pCache, rConSec, &bNoAnswer); + del_DasNodeWalk(pWalk); + if(pRoot != NULL) return pRoot; + if(!bNoAnswer) break; + rConSec *= DASHTTP_TO_MULTI; + } + daslog_error("No mirror of the das catalog index answered"); return NULL; } -/* ************************************************************************* */ -/* Construction */ +static DasNode* _DasCat_keep(DasCat* pThis, DasNode* pRoot); /* below, with the session */ -DasNode* _DasNode_newUrl( - const char* sUrl, const char* sPathUri, DasCredMngr* pMgr, const char* sAgent, - const DasNodeCache* pCache -){ - /* Every node has a URI; one loaded on its own is named by where it is */ - char sUri[600]; - if((sPathUri == NULL)||(sPathUri[0] == '\0')) _das_uri_local(sUrl, sUri, sizeof(sUri)); - else _das_uri_expand(sPathUri, sUri, sizeof(sUri)); - - /* Since they want a specific URL, go for the max timeout */ - DasNode* pNode = _DasNode_mkNode(sUrl, sUri, pMgr, sAgent, DASHTTP_TO_MAX, pCache); - if(pNode) pNode->bOwner = true; - return pNode; +/* The seed given to DasCat_setSeed(), loaded once and kept */ +static DasNode* _DasCat_seedNode(DasCat* pThis) +{ + if(pThis->pSeed == NULL) + pThis->pSeed = _DasCat_keep(pThis, _dascat_loadUrl( + pThis->sSeedUrl, pThis->sSeedUri, pThis->pMgr, pThis->sAgent, pThis->pCache + )); + return pThis->pSeed; } -DasNode* new_DasNode_url( - const char* sUrl, const char* sPathUri, DasCredMngr* pMgr, const char* sAgent -){ - return _DasNode_newUrl(sUrl, sPathUri, pMgr, sAgent, NULL); +/* The session's copy of the compiled in seed, loaded once and kept, as + * das2py has always done. Every federated lookup is then a walk from it + * and the levels walked through stay loaded for the next one. */ +static DasNode* _DasCat_das2(DasCat* pThis) +{ + if(pThis->pDas2 == NULL) + pThis->pDas2 = _DasCat_keep(pThis, _dascat_loadSeed(pThis->pMgr, pThis->sAgent, pThis->pCache)); + return pThis->pDas2; } -DasNode* _DasNode_new( - const char* sPathUri, DasCredMngr* pMgr, const char* sAgent, +/* One node from a URL or file, named by the caller or by its location. + * Since they want a specific URL, go for the max timeout. */ +static DasNode* _dascat_loadUrl( + const char* sUrl, const char* sPathUri, DasCredMngr* pMgr, const char* sAgent, const DasNodeCache* pCache ){ - /* Find the node using the path, start at the root */ - DasNode* pTop = NULL; - DasNode* pNode = NULL; - - /* Expanding shorthands here, in the library, means every client gets - the same spelling. Names under local: belong to nodes loaded by - location in some session; the federated roots have never heard of - them and asking would only cost a round trip. */ - char sFullUri[600] = {'\0'}; - if(sPathUri != NULL){ - sPathUri = _das_uri_expand(sPathUri, sFullUri, sizeof(sFullUri)); - if(_das_uri_isLocal(sPathUri)){ - daslog_debug_v("%s is a local: name, those are not routed through the federated catalog", sPathUri); - return NULL; - } - } + char sUri[600] = {'\0'}; + if((sPathUri != NULL) && (sPathUri[0] != '\0')) _das_uri_expand(sPathUri, sUri, sizeof(sUri)); - /* Changing node lookup and time outs. New strategy, try to contact a node - if it doesn't work within the time out, try the next one. If all fail - then increase the time out and try again. - */ - - size_t uRoots = 0; - const char** aRoots = das_root_urls(&uRoots); - float rConSec = DASHTTP_TO_MIN; - - while((rConSec <= DASHTTP_TO_MAX )&&(pNode == NULL)){ - for(size_t i = 0; i < uRoots; ++i){ - /* Credentials should never be needed for top level nodes */ - pTop = _DasNode_mkNode(aRoots[i], NULL, NULL, sAgent, rConSec, pCache); - if(pTop == NULL) continue; - if(pTop->nType != d2node_catalog) continue; - - if(sPathUri == NULL){ /* They may be looking for a root node */ - pNode = pTop; - break; - } - - /* Depends on the fact that top node is a das catalog */ - pNode = _DasNode_loadSubNode_dasCat((DasCatNode*)pTop, sPathUri, pMgr, sAgent); - if(pNode != NULL) break; + struct dascat_fetch f; + _dascat_fetch(pMgr, sAgent, pCache, sUrl, DASHTTP_TO_MAX, &f); + DasNode* pNode = NULL; + if(f.pBytes != NULL){ + size_t uLen = 0; + const char* p = (const char*)DasAry_getBytesIn(f.pBytes, DIM0, &uLen); + pNode = _DasNode_fromJson(p, uLen, sUrl, f.sFinal, sUri); + if(pNode != NULL){ + pNode->bOwner = true; + _dascat_keep(&f, p, uLen); } - rConSec *= DASHTTP_TO_MULTI; - } - - if(pNode == NULL){ - daslog_error_v("Failed to load the requested node %s from any of the " - "built in catalog trees", sPathUri); - return NULL; } - - /* The caller gets this node, so mark it theirs before freeing the walk - * above it; _DasNode_delTree() then leaves it alone even when - * pTop == pNode */ - pNode->bOwner = true; - pNode->pMgr = pMgr; /* pTop was fetched without them, the kept node - hands them to whatever it loads later */ - pNode->sAgent = sAgent; - pNode->pCache = pCache; - _DasNode_delTree(pTop); - + _dascat_fetchDone(&f); return pNode; } -DasNode* new_DasNode(const char* sPathUri, DasCredMngr* pMgr, const char* sAgent) -{ - return _DasNode_new(sPathUri, pMgr, sAgent, NULL); -} - - /* ************************************************************************* */ /* Small helpers */ -static bool _pfx(const char* s, const char* sPrefix) -{ - return (s != NULL) && (strncmp(s, sPrefix, strlen(sPrefix)) == 0); -} - static void _setStr(char** ps, const char* s) { free(*ps); @@ -724,41 +534,21 @@ static char* _expand(const char* sIn) return das_strdup(_das_uri_expand(sIn, sBuf, sizeof(sBuf))); } -bool das_isUri(const char* sPath) -{ - if((sPath == NULL) || !isalpha((unsigned char)sPath[0])) return false; - const char* p = sPath + 1; - while(isalnum((unsigned char)*p) || (*p == '+') || (*p == '-') || (*p == '.')) ++p; - /* A single letter before the colon is a Windows drive, not a scheme */ - return (*p == ':') && (p > sPath + 1); -} - -bool das_isUrlOrFile(const char* sPath) -{ - if(sPath == NULL) return false; - if(_pfx(sPath, "http://")||_pfx(sPath, "https://")||_pfx(sPath, "file://")) return true; - if((sPath[0] == '.')||(sPath[0] == '/')||(sPath[0] == '\\')) return true; - if(isalpha((unsigned char)sPath[0]) && (sPath[1] == ':') && ((sPath[2] == '\\')||(sPath[2] == '/'))) - return true; /* a Windows drive, catalog URIs never have their first colon at index 1 */ - size_t u = strlen(sPath); - return (u > 5) && (strcmp(sPath + u - 5, ".json") == 0); -} - -/* Take ownership of a node a constructor returned */ +/* Take ownership of a tree top for the life of the session */ static DasNode* _DasCat_keep(DasCat* pThis, DasNode* pRoot) { if(pRoot == NULL) return NULL; - if(pThis->uRoots == pThis->uRootCap){ - size_t uCap = (pThis->uRootCap == 0) ? 8 : 2 * pThis->uRootCap; - DasNode** a = (DasNode**)realloc(pThis->aRoots, uCap * sizeof(DasNode*)); + if(pThis->uOwned == pThis->uOwnedCap){ + size_t uCap = (pThis->uOwnedCap == 0) ? 8 : 2 * pThis->uOwnedCap; + DasNode** a = (DasNode**)realloc(pThis->aOwned, uCap * sizeof(DasNode*)); if(a == NULL){ - das_error(DASERR_CAT, "Couldn't grow the catalog root list"); - return pRoot; /* leaks the root rather than losing the node */ + das_error(DASERR_CAT, "Couldn't grow the session's node list"); + return pRoot; /* leaks the node rather than losing it */ } - pThis->aRoots = a; - pThis->uRootCap = uCap; + pThis->aOwned = a; + pThis->uOwnedCap = uCap; } - pThis->aRoots[pThis->uRoots++] = pRoot; + pThis->aOwned[pThis->uOwned++] = pRoot; return pRoot; } @@ -794,9 +584,9 @@ void del_DasCat(DasCat* pThis) if(pThis == NULL) return; /* Nodes borrow the credentials manager and the cache settings, so they go - * first. The policy root is in aRoots like every other root. */ - for(size_t u = 0; u < pThis->uRoots; ++u) del_DasNode(pThis->aRoots[u]); - free(pThis->aRoots); + * first. The mount table and the seeds are in aOwned like every tree top. */ + for(size_t u = 0; u < pThis->uOwned; ++u) del_DasNode(pThis->aOwned[u]); + free(pThis->aOwned); if(pThis->pMgr != NULL) del_CredMngr(pThis->pMgr); free(pThis->pCache); @@ -809,6 +599,10 @@ void del_DasCat(DasCat* pThis) free(pThis->aMounts); free(pThis->sLastPath); free(pThis->sLastUri); + free(pThis->sNear); + free(pThis->sNearRest); + free(pThis->sSeedUrl); + free(pThis->sSeedUri); free(pThis); } @@ -833,7 +627,7 @@ void DasCat_setCache(DasCat* pThis, const char* sDir, int nMaxAgeSec) pThis->pCache->sDir[0] = '\0'; return; } - /* sDir may be our own buffer, from DasCat_cacheDir(), to change the age */ + /* sDir may be our buffer, from DasCat_cacheDir(), to change the age */ if(sDir != pThis->pCache->sDir) strcpy(pThis->pCache->sDir, sDir); pThis->pCache->nMaxAgeSec = nMaxAgeSec; } @@ -853,9 +647,15 @@ int DasCat_cacheAge(const DasCat* pThis) /* Root policy */ /* Roots are built on first use, so the policy is frozen by then */ +/* das_strdup() gives NULL for "", and these fields promise a string */ +static char* _strdupz(const char* s) +{ + return ((s == NULL)||(s[0] == '\0')) ? (char*)calloc(1, 1) : das_strdup(s); +} + static bool _DasCat_frozen(const DasCat* pThis) { - if(pThis->uRoots == 0) return false; + if(pThis->uOwned == 0) return false; daslog_error("Catalog policy can't change after a path has been resolved"); return true; } @@ -863,6 +663,10 @@ static bool _DasCat_frozen(const DasCat* pThis) DasErrCode DasCat_setPrefix(DasCat* pThis, const char* sUri) { if(_DasCat_frozen(pThis)) return DASERR_CAT; + if(pThis->bSeed){ + daslog_error_v("Relative paths start at the seed, %s", pThis->sSeedUri[0] ? pThis->sSeedUri : "a root"); + return DASERR_CAT; + } if((sUri == NULL)||(sUri[0] == '\0')){ daslog_error("A path prefix can't be empty"); return DASERR_CAT; @@ -885,12 +689,15 @@ DasErrCode DasCat_setPrefix(DasCat* pThis, const char* sUri) return DAS_OKAY; } -const char* DasCat_prefix(const DasCat* pThis){ return pThis->sPrefix; } +const char* DasCat_prefix(const DasCat* pThis) +{ + return pThis->bSeed ? pThis->sSeedUri : pThis->sPrefix; +} -bool DasCat_hasRoot(const DasCat* pThis){ return pThis->bIsolated; } +bool DasCat_hasSeed(const DasCat* pThis){ return pThis->bSeed; } /* The mount whose URI covers sUri, or -1. Longest match, as the mount - * root itself resolves, though mounts rarely nest. */ + * table itself resolves, though mounts rarely nest. */ static int _DasCat_mountFor(const DasCat* pThis, const char* sUri) { int nBest = -1; @@ -905,7 +712,7 @@ static int _DasCat_mountFor(const DasCat* pThis, const char* sUri) return nBest; } -/* Add to the mount list. An empty URI names the node by its location. */ +/* Add to the mount list, the URI already checked */ static DasErrCode _DasCat_addMount(DasCat* pThis, const char* sUrl, const char* sUri) { struct das_cat_mount* a = (struct das_cat_mount*)realloc( @@ -914,18 +721,12 @@ static DasErrCode _DasCat_addMount(DasCat* pThis, const char* sUrl, const char* if(a == NULL) return das_error(DASERR_CAT, "Couldn't grow the mount list"); pThis->aMounts = a; a[pThis->uMounts].sUrl = das_strdup(sUrl); - if((sUri == NULL)||(sUri[0] == '\0')){ - char sBuf[600]; - a[pThis->uMounts].sUri = das_strdup(_das_uri_local(sUrl, sBuf, sizeof(sBuf))); - } - else{ - a[pThis->uMounts].sUri = _expand(sUri); - } + a[pThis->uMounts].sUri = _expand(sUri); pThis->uMounts += 1; return DAS_OKAY; } -/* The checks shared by roots and mounts */ +/* The checks shared by seeds and mounts */ static bool _DasCat_locationOk(const char* sUrl, const char* sUri, bool bUriNeeded) { if(!das_isUrlOrFile(sUrl)){ @@ -944,29 +745,23 @@ static bool _DasCat_locationOk(const char* sUrl, const char* sUri, bool bUriNeed return true; } -DasErrCode DasCat_setRoot(DasCat* pThis, const char* sUrl, const char* sUri) +DasErrCode DasCat_setSeed(DasCat* pThis, const char* sUrl, const char* sUri) { if(_DasCat_frozen(pThis)) return DASERR_CAT; - if(pThis->bIsolated){ - daslog_error("A catalog has one root, and one is already set"); + if(pThis->bSeed){ + daslog_error("A session has one seed, and one is already set"); return DASERR_CAT; } + if(pThis->bPrefixGiven) + daslog_info_v("Relative paths started at %s, now they start at the seed", pThis->sPrefix); if(!_DasCat_locationOk(sUrl, sUri, false)) return DASERR_CAT; - if(_DasCat_addMount(pThis, sUrl, sUri) != DAS_OKAY) return DASERR_CAT; - pThis->uRoot = pThis->uMounts - 1; - pThis->bIsolated = true; + pThis->sSeedUrl = das_strdup(sUrl); + pThis->sSeedUri = ((sUri != NULL) && (sUri[0] != '\0')) ? _expand(sUri) : (char*)calloc(1, 1); + pThis->bSeed = true; return DAS_OKAY; } -const char* DasCat_rootUrl(const DasCat* pThis) -{ - return pThis->bIsolated ? pThis->aMounts[pThis->uRoot].sUrl : NULL; -} - -const char* DasCat_rootUri(const DasCat* pThis) -{ - return pThis->bIsolated ? pThis->aMounts[pThis->uRoot].sUri : NULL; -} +const char* DasCat_seedUrl(const DasCat* pThis){ return pThis->sSeedUrl; } DasErrCode DasCat_mount(DasCat* pThis, const char* sUrl, const char* sUri) { @@ -980,18 +775,19 @@ size_t DasCat_mountCount(const DasCat* pThis){ return pThis->uMounts; } /* ************************************************************************* */ /* Resolution */ -/* The mount root, loaded on first use. With a root set the mounts are the - * whole catalog, otherwise unclaimed paths fall through to the federated - * roots. */ -static DasNode* _DasCat_root(DasCat* pThis) +/* The mount table, built on first use: a synthetic node whose children + * are the mounts by URI, longest URI first so a mount under another + * shadows it. It never falls through to the federated catalog; that is + * decided here, see _DasCat_byUri(). */ +static DasNode* _DasCat_table(DasCat* pThis) { - if(pThis->pRoot != NULL) return pThis->pRoot; + if(pThis->pTable != NULL) return pThis->pTable; if(pThis->uMounts == 0) return NULL; const char** aUrl = (const char**)calloc(pThis->uMounts, sizeof(char*)); const char** aUri = (const char**)calloc(pThis->uMounts, sizeof(char*)); if((aUrl == NULL)||(aUri == NULL)){ - das_error(DASERR_CAT, "Couldn't allocate the mount root"); + das_error(DASERR_CAT, "Couldn't allocate the mount table"); free(aUrl); free(aUri); return NULL; } @@ -999,7 +795,7 @@ static DasNode* _DasCat_root(DasCat* pThis) aUrl[u] = pThis->aMounts[u].sUrl; aUri[u] = pThis->aMounts[u].sUri; } - /* The mount root matches children in document order, so a mount at + /* The table matches children in document order, so a mount at site:/uiowa listed before one at site:/uiowa/juno/wav would shadow it. Longest URI first makes the order mounts were given in irrelevant, as it should be: lists in a catalog are never @@ -1010,73 +806,90 @@ static DasNode* _DasCat_root(DasCat* pThis) s = aUrl[j]; aUrl[j] = aUrl[j-1]; aUrl[j-1] = s; } } - pThis->pRoot = _DasNode_newMountRoot( - aUrl, aUri, pThis->uMounts, !pThis->bIsolated, pThis->pMgr, pThis->sAgent, - pThis->pCache - ); + pThis->pTable = _DasNode_newMountTable(aUrl, aUri, pThis->uMounts); free(aUrl); free(aUri); - return _DasCat_keep(pThis, pThis->pRoot); + return _DasCat_keep(pThis, pThis->pTable); } -/* A local: name reaches only the session's own mounts, see D2URI_LOCAL */ -static bool _DasCat_unrouted(const DasCat* pThis, const char* sUri) +/* A full catalog URI under the current policy, sLastUri already set. + * Mounts are opportunistic: a URI one covers is walked from the mount + * table, anything else from the seed, or from the compiled in seed when + * none is set. Shared by DasCat_resolve() and DasCat_nearest(), which + * has a URI in hand and must not go through the location test. */ +static DasNode* _DasCat_byUri(DasCat* pThis, const char* sUri) { - if(!_das_uri_isLocal(sUri) || (_DasCat_mountFor(pThis, sUri) >= 0)) return false; - daslog_debug_v("%s is a local: name and nothing is mounted there", sUri); - return true; + if(_DasCat_mountFor(pThis, sUri) >= 0){ + DasNode* pTable = _DasCat_table(pThis); + if(pTable == NULL) return NULL; + return _dascat_walkFrom(pThis, pTable, sUri); + } + if(pThis->bSeed){ + if(!DasCat_covers(pThis, sUri)){ + daslog_debug_v("%s is outside the seed %s", sUri, pThis->sSeedUri[0] ? pThis->sSeedUri : "(a root)"); + return NULL; + } + DasNode* pSeed = _DasCat_seedNode(pThis); + if(pSeed == NULL) return NULL; + /* The seed itself is asked for by its prefix; it may be a source, + which no walk can start from, so it is handed back directly */ + const char* sRest = sUri + strlen(pThis->sSeedUri); + if(sRest[0] == '\0') return pSeed; + return _dascat_walkFrom(pThis, pSeed, sRest); + } + DasNode* pDas2 = _DasCat_das2(pThis); + if(pDas2 == NULL) return NULL; + return _dascat_walkFrom(pThis, pDas2, sUri); } DasNode* DasCat_resolve(DasCat* pThis, const char* sPath) { bool bNoPath = (sPath == NULL)||(sPath[0] == '\0'); _setStr(&pThis->sLastPath, bNoPath ? NULL : sPath); - free(pThis->sLastUri); - pThis->sLastUri = NULL; + free(pThis->sLastUri); pThis->sLastUri = NULL; + free(pThis->sNear); pThis->sNear = NULL; + free(pThis->sNearRest); pThis->sNearRest = NULL; - /* A node by location, named under local: */ + /* A node by location: a root, the location stands in for a + name in messages */ if(!bNoPath && das_isUrlOrFile(sPath)){ - char sUri[600]; - pThis->sLastUri = das_strdup(_das_uri_local(sPath, sUri, sizeof(sUri))); - return _DasCat_keep(pThis, _DasNode_newUrl( - sPath, pThis->sLastUri, pThis->pMgr, pThis->sAgent, pThis->pCache + pThis->sLastUri = das_strdup(sPath); + return _DasCat_keep(pThis, _dascat_loadUrl( + sPath, NULL, pThis->pMgr, pThis->sAgent, pThis->pCache )); } - if(pThis->uMounts > 0){ - DasNode* pRoot = _DasCat_root(pThis); - if(pRoot == NULL) return NULL; - const char* sTopUri = pThis->aMounts[pThis->bIsolated ? pThis->uRoot : 0].sUri; - - /* No path: the top node itself. A full URI, or a relative path - * under an explicit prefix: the mount root decides whether a mount - * or the federated catalog owns it. Anything else is relative to - * the top node. */ + /* With a seed, relative paths start at it; a full URI goes through the + * mounts or nowhere */ + if(pThis->bSeed){ if(bNoPath){ - pThis->sLastUri = das_strdup(sTopUri); - return DasNode_subNode(pRoot, sTopUri, NULL, NULL); + pThis->sLastUri = _strdupz(pThis->sSeedUri); + return _DasCat_seedNode(pThis); } - if(pThis->bPrefixGiven && !das_isUri(sPath)) - pThis->sLastUri = das_string("%s%s", pThis->sPrefix, sPath); - else + if(das_isUri(sPath)){ pThis->sLastUri = _expand(sPath); - if(das_isUri(pThis->sLastUri)){ - if(_DasCat_unrouted(pThis, pThis->sLastUri)) return NULL; - return DasNode_subNode(pRoot, pThis->sLastUri, NULL, NULL); + return _DasCat_byUri(pThis, pThis->sLastUri); } - - DasNode* pTop = DasNode_subNode(pRoot, sTopUri, NULL, NULL); - if(pTop == NULL){ - _setStr(&pThis->sLastUri, sTopUri); + DasNode* pSeed = _DasCat_seedNode(pThis); + if(pSeed == NULL){ + pThis->sLastUri = _strdupz(pThis->sSeedUri); return NULL; } - const char* sSep = DasNode_isCatalog(pTop) ? DasNode_pathSep(pTop) : "/"; - free(pThis->sLastUri); - pThis->sLastUri = das_string("%s%s%s", sTopUri, sSep ? sSep : "/", sPath); - return DasNode_subNode(pTop, sPath, NULL, NULL); + /* Names below a root are the path from it; below a prefix the + seed's separator joins them. The full name then goes through + the same door as a URI, so a mount under the seed is consulted + whichever way the path was spelled. */ + if(pThis->sSeedUri[0] == '\0'){ + pThis->sLastUri = das_strdup(sPath); + } + else{ + const char* sSep = DasNode_isCatalog(pSeed) ? DasNode_pathSep(pSeed) : "/"; + pThis->sLastUri = das_string("%s%s%s", pThis->sSeedUri, sSep ? sSep : "/", sPath); + } + return _DasCat_byUri(pThis, pThis->sLastUri); } - /* The federated catalog under the prefix */ + /* Otherwise relative paths start at the prefix, mounts or not */ if(bNoPath){ /* The prefix with its trailing separator removed names the container */ pThis->sLastUri = das_strdup(pThis->sPrefix); @@ -1089,65 +902,79 @@ DasNode* DasCat_resolve(DasCat* pThis, const char* sPath) else pThis->sLastUri = das_string("%s%s", pThis->sPrefix, sPath); - if(_DasCat_unrouted(pThis, pThis->sLastUri)) return NULL; - return _DasCat_keep(pThis, _DasNode_new( - pThis->sLastUri, pThis->pMgr, pThis->sAgent, pThis->pCache - )); + return _DasCat_byUri(pThis, pThis->sLastUri); } const char* DasCat_lastUri(const DasCat* pThis){ return pThis->sLastUri; } bool DasCat_covers(const DasCat* pThis, const char* sUri) { - /* local: names go nowhere but a mount; otherwise mounts fall through to - * the federated catalog and only a root walls paths off */ - if(_das_uri_isLocal(sUri) || pThis->bIsolated) return _DasCat_mountFor(pThis, sUri) >= 0; - return true; + /* Mounts fall through to the federated catalog; only a seed walls paths + * off. Names below a root seed are paths from it, so anything with a + * scheme in front is outside it. */ + if(!pThis->bSeed) return true; + if(_DasCat_mountFor(pThis, sUri) >= 0) return true; + if(pThis->sSeedUri[0] == '\0') return !das_isUri(sUri); + return strncmp(sUri, pThis->sSeedUri, strlen(pThis->sSeedUri)) == 0; } DasNode* DasCat_nearest(DasCat* pThis, const char** psRest) { if(psRest != NULL) *psRest = NULL; - if((pThis->sLastPath == NULL) || das_isUrlOrFile(pThis->sLastPath)) return NULL; - - /* The resolves below overwrite the last path and URI, take them back - * so the caller's view of the failure survives a miss here */ - char* sOrigPath = pThis->sLastPath; - char* sOrigUri = pThis->sLastUri; - pThis->sLastPath = NULL; - pThis->sLastUri = NULL; - - char* sParent = das_strdup(sOrigPath); - DasNode* pNode = NULL; - size_t uCut = 0; - while(true){ - size_t u = strlen(sParent); - while((u > 0) && ((sParent[u-1] == '/')||(sParent[u-1] == ':'))) sParent[--u] = '\0'; - char* pLast = strrchr(sParent, '/'); - if((pLast == NULL)||(pLast == sParent)) break; - *pLast = '\0'; - uCut = (size_t)(pLast - sParent); /* sParent is a prefix copy of sOrigPath */ - DasNode* p = DasCat_resolve(pThis, sParent); - if((p != NULL) && DasNode_isCatalog(p)){ pNode = p; break; } - } - free(sParent); - free(pThis->sLastPath); - pThis->sLastPath = sOrigPath; - - if(pNode == NULL){ + if(pThis->pNearNode != NULL){ + /* A root: nothing to resolve again, and no URI to report, so the + last path and URI say so with an empty string */ + DasNode* pNode = pThis->pNearNode; + pThis->pNearNode = NULL; + _setStr(&pThis->sLastPath, NULL); free(pThis->sLastUri); - pThis->sLastUri = sOrigUri; + pThis->sLastUri = (char*)calloc(1, 1); + if(psRest != NULL) *psRest = pThis->sNearRest; + return pNode; + } + if(pThis->sNear == NULL) return NULL; + + /* The walk that failed said how far it got. Loading that container + * again is cheap, loaded children or the node cache, and it has the + * documented effect on the last path and URI: afterwards they name the + * nearest node. The walk's memory is taken first, the resolve clears + * it. */ + char* sNear = pThis->sNear; pThis->sNear = NULL; + char* sRest = pThis->sNearRest; pThis->sNearRest = NULL; + char* sOrigPath = pThis->sLastPath; pThis->sLastPath = das_strdup(sNear); + char* sOrigUri = pThis->sLastUri; pThis->sLastUri = sNear; + + DasNode* pNode = _DasCat_byUri(pThis, pThis->sLastUri); + if(pNode == NULL){ + /* It loaded a moment ago, so a server went away between calls; + * keep the caller's view of the original failure */ + free(sRest); + free(pThis->sLastPath); pThis->sLastPath = sOrigPath; + free(pThis->sLastUri); pThis->sLastUri = sOrigUri; return NULL; } + free(sOrigPath); free(sOrigUri); - if(psRest != NULL){ - const char* s = sOrigPath + uCut; - while((*s == '/')||(*s == ':')) ++s; - *psRest = s; - } + free(pThis->sNearRest); + pThis->sNearRest = sRest; /* the caller's pointer, valid until the next resolve */ + if(psRest != NULL) *psRest = sRest; return pNode; } +DasNode* DasCat_seed(DasCat* pThis) +{ + return pThis->bSeed ? _DasCat_seedNode(pThis) : _DasCat_das2(pThis); +} + +DasNode* DasCat_subNode(DasCat* pThis, DasNode* pNode, const char* sRelPath) +{ + if(pNode == NULL){ + daslog_error("No node to load a child of"); + return NULL; + } + return _dascat_walkFrom(pThis, pNode, sRelPath); +} + /* ************************************************************************* */ /* Sources */ diff --git a/das3/catalog.h b/das3/catalog.h index e32fa13..e6d326e 100644 --- a/das3/catalog.h +++ b/das3/catalog.h @@ -15,8 +15,8 @@ * version 2.1 along with das2C; if not, see . */ -/** @file catalog.h A catalog session: roots, credentials and node memory - * in one object, with mountable sub-trees */ +/** @file catalog.h A catalog session: where paths start, credentials and + * node memory in one object, with mounts */ #ifndef _das_catalog_h_ #define _das_catalog_h_ @@ -44,10 +44,11 @@ extern "C" { * * @par Names * - * Every node has a globally unique URI, composed by walking down from the - * root: each container appends its separator and the child's name to its - * own URI. A node does not know its name, the walk gives it one. The - * result looks like a file path with a URI scheme in front: + * A node's URI is composed by walking down from the root: each container + * appends its separator and the child's name to its URI. A node does + * not know its name, the walk gives it one, and the root, which has + * nothing above it, has none. The result looks like a file path with a + * URI scheme in front: * * tag:das2.org,2012:site:/uiowa/juno/wav/survey * @@ -72,8 +73,8 @@ extern "C" { * tag:das2.org,2012:site:/uiowa/juno/wav/survey * * and walking proceeds from there as if the file had been fetched from - * the catalog, see DasCat_mount() and DasCat_setRoot(). A file loaded - * with no name at all is named after its location under D2URI_LOCAL. + * the catalog, see DasCat_mount() and DasCat_setSeed(). A file loaded + * with no name at all is a root and has no URI. * * @par Reading a node * @@ -96,14 +97,27 @@ extern "C" { * source.h. * * Two objects do the work. DasCat, in das3/catalog.h, is a session: it - * holds the root policy (where relative paths start, see DasCat_setRoot() - * and DasCat_mount()), the credentials and user agent every load uses, - * and the memory of every node it loads. DasNode, in das3/node.h, is one - * node: its type, URI, source URL and parsed document, with - * DasNode_subNode() to step down to a child. Most programs only ever - * construct a DasCat. The node constructors in node.h are the lower - * level interface for programs that manage roots themselves; see - * "Working with nodes directly" in node.h. + * holds where relative paths start, see DasCat_setPrefix() and + * DasCat_setSeed(), any mounts, the credentials and user agent every load + * uses, and the memory of every node it loads. DasNode, in das3/node.h, + * is one node: its type, URI, source URL and parsed document, with + * DasNodeWalk to step down to a child. Most programs only ever construct + * a DasCat. The node interface is for programs that fetch for themselves, + * see "Nodes without a session" in node.h. + * + * @par Seeds, mounts, and the root + * + * A seed is a node a program starts from with nothing above it in memory. + * The compiled in seed is the index of the das catalog, see + * das_seed_urls(); a session loads it on first use and keeps it, so the + * tree below it grows as paths are resolved. A file can be a seed too, + * see DasCat_setSeed(), given a prefix that names it, or none. A seed + * given no prefix is a root: the one node with no URI, from which names + * are composed starting from nothing. A mount, see DasCat_mount(), is a + * node grafted at a name in the session's name space, whichever seed + * defines it: a path from a root, a URI under a prefix, or a federated + * URI when no seed is set. It is consulted before the seed for anything + * under that name, however the path was spelled. */ /** @addtogroup catalog @@ -121,7 +135,7 @@ extern "C" { * A DasCat turns catalog paths into nodes. It remembers where relative * paths start, which credentials and user agent to send, and every node it * has loaded, so a program, a MATLAB extension or a Python module can say - * "resolve this path" and get a node back without managing roots itself. + * "resolve this path" and get a node back without managing seeds itself. * * Paths given to DasCat_resolve() are catalog paths, not web paths, in any * of these forms: @@ -130,22 +144,22 @@ extern "C" { * - the shorthands "site:/uiowa/..." and "test:/..." * - a full catalog URI: "tag:das2.org,2012:site:/uiowa/..." * - a node by location, a URL or a file: "https://host/node.json", - * "./node.json". Such a node is named after its location under - * D2URI_LOCAL, since every node has a URI. - * - * Where relative paths start is the root policy, and the catalog tree is - * navigated like a file system, so the policy borrows its words: - * - * - By default relative paths start at DASCAT_DEF_PREFIX in the - * federated catalog. DasCat_setPrefix() moves that. - * - DasCat_mount() puts a node, loaded by location, at a catalog URI - * on top of the federated catalog. Paths under the URI resolve - * through the mounted node, everything else through the compiled in - * roots as usual. Several nodes may be mounted. - * - DasCat_setRoot() mounts a node and makes the mounts the whole - * catalog: relative paths start at that node and nothing outside - * the mounts can be reached. This is how an offline catalog is - * assembled from a few files. + * "./node.json". Such a node is a root, with no URI; + * DasCat_lastUri() reports the location in its place. + * + * Where relative paths start is one setting with two ways to set it: + * + * - By default they start at DASCAT_DEF_PREFIX in the federated + * catalog. DasCat_setPrefix() moves that. + * - DasCat_setSeed() makes a node loaded by location the seed: relative + * paths start at it, full URIs resolve only under its prefix or under + * a mount, and nothing else can be reached. This is how an offline + * catalog is assembled from a few files. + * + * DasCat_mount() puts a node, loaded by location, at a catalog URI over + * the federated catalog. Paths under the URI resolve through the mounted + * node, everything else through the compiled in seed as usual. Mounts + * never move where relative paths start. * * Policy is set before the first resolve. Nodes returned by * DasCat_resolve() are owned by the catalog and live until del_DasCat(). @@ -163,32 +177,10 @@ extern "C" { */ typedef struct das_cat DasCat; -/** Is this string a node location rather than a catalog path? - * - * True for http://, https:// and file:// URLs, for anything starting with - * '.', '/', '\' or a drive letter, and for anything ending in ".json". All else - * is taken to be a catalog path or URI. This is the rule DasCat_resolve() - * uses, published so command line parsers can agree with it. - * - * @memberof DasCat - */ -DAS_API bool das_isUrlOrFile(const char* sPath); - -/** Is this string a full URI rather than a relative catalog path? - * - * True when it starts with a URI scheme, a letter followed by letters, - * digits, '+', '-' or '.' and then a colon, as in "tag:", "doi:" and the - * shorthands "site:" and "test:". A single letter before the colon is a - * Windows drive, not a scheme. Note that URLs are URIs too; test with - * das_isUrlOrFile() first when the difference matters. - * - * @memberof DasCat - */ -DAS_API bool das_isUri(const char* sPath); /** Create a catalog session * - * The returned catalog it's node cache enabled right out of the gate, + * The returned session has its node cache enabled right out of the gate, * to turn it off see DasCat_setCache() * * @param pMgr Credentials manager consulted when a node or data source asks @@ -228,8 +220,8 @@ DAS_API const char* DasCat_agent(const DasCat* pThis); * asked whether it changed, using the ETag or Last-Modified it came with, * and only a changed node is downloaded again. A server that cannot be * reached leaves the old copy in use, with a warning. Nodes read from - * local files are not cached, and neither are nodes made through the - * constructors in node.h + * local files are not cached, and neither are nodes built from bytes a + * program fetched itself, see new_DasNode_json(). * * @param sDir Directory for the copies, created on first use. NULL or an * empty string turns caching off for this session. Passing @@ -258,88 +250,100 @@ DAS_API int DasCat_cacheAge(const DasCat* pThis); * The shorthands "site:" and "test:" are expanded; given alone they name * the container and its ":/" child separator is supplied, so "site:" is * the same prefix as "tag:das2.org,2012:site:/". A URL or file is - * refused, those are roots, see DasCat_setRoot(). + * refused; a node to start from is a seed, see DasCat_setSeed(). * * @param sUri A catalog URI prefix, typically ending in the separator the * container uses, such as "tag:das2.org,2012:test:/" - * @returns DAS_OKAY, or DASERR_CAT with the reason logged + * @returns DAS_OKAY, or DASERR_CAT with the reason logged: a location, + * an empty string, or a seed already set, since relative paths + * then start at the seed * @memberof DasCat */ DAS_API DasErrCode DasCat_setPrefix(DasCat* pThis, const char* sUri); -/** The current prefix, DASCAT_DEF_PREFIX unless DasCat_setPrefix() was called +/** Where relative paths start: the seed's URI when one is set, else the + * prefix, DASCAT_DEF_PREFIX unless DasCat_setPrefix() was called * @memberof DasCat */ DAS_API const char* DasCat_prefix(const DasCat* pThis); -/** Mount a node and make the mounts the whole catalog +/** Make a node loaded by location the seed, and the mounts the whole catalog * - * Relative paths then start at this node, full URIs resolve only when a - * mount covers them, and everything else fails, which DasCat_covers() can - * tell apart from an ordinary miss. Further DasCat_mount() calls, before - * or after, add to the offline catalog. Refused once a root is set. + * Relative paths then start at this node, full URIs resolve only under + * its prefix or under a mount, and everything else fails, which + * DasCat_covers() can tell apart from an ordinary miss. Further DasCat_mount() calls, before + * or after, add to the offline catalog. A session has one seed, and one + * place relative paths start: a prefix set earlier is superseded, and + * DasCat_setPrefix() is refused afterwards. * - * The URI declares what the node is: a development server's copy of a + * The prefix declares what the node is: a development server's copy of a * catalog branch stands for the branch's federated name, so requisitions - * made against it are good against the live catalog too. Without a URI - * the node is named after its location under D2URI_LOCAL. That is fine - * for a one off file, but a requisition stamped with a local: name will - * not match the catalog node the file was copied from; give the real URI - * when it has one. + * made against it are good against the live catalog too. Without a + * prefix the seed is a root: it has no URI, nodes below it are named by + * their path from it, and requisitions made against sources below it + * carry those names. Give the real URI when the file stands in for a + * catalog node, so that requisitions match it. * - * @param sUrl A URL or file path that new_DasNode_url() accepts + * @param sUrl A URL or file path of a node document * @param sUri The catalog URI the node has, shorthands accepted, or NULL - * to name it by location + * for a root * @returns DAS_OKAY, or DASERR_CAT with the reason logged * @memberof DasCat */ -DAS_API DasErrCode DasCat_setRoot(DasCat* pThis, const char* sUrl, const char* sUri); +DAS_API DasErrCode DasCat_setSeed(DasCat* pThis, const char* sUrl, const char* sUri); -/** The root location given to DasCat_setRoot(), or NULL if none is set +/** The location given to DasCat_setSeed(), or NULL if none was * @memberof DasCat */ -DAS_API const char* DasCat_rootUrl(const DasCat* pThis); +DAS_API const char* DasCat_seedUrl(const DasCat* pThis); -/** The root's catalog URI, or NULL if the root is nameless or not set +/** The seed node, loading it if need be + * + * The node given to DasCat_setSeed(), or the index of the das catalog, + * a root with no URI whose children are the top level catalogs. Owned + * by the session. For programs that list the catalog from the top or + * walk it themselves with DasCat_subNode(). + * + * @returns the node, or NULL after logging why not * @memberof DasCat */ -DAS_API const char* DasCat_rootUri(const DasCat* pThis); +DAS_API DasNode* DasCat_seed(DasCat* pThis); -/** Put a node, loaded by location, at a catalog URI over the federated catalog +/** Put a node, loaded by location, at a name over the catalog * - * Paths under sUri load from sUrl and shadow whatever the federated - * catalog says for them, everything else resolves through the compiled in - * roots as usual. May be called more than once; when one mount point is - * under another the longer URI wins. Relative paths start at the first - * mount unless DasCat_setPrefix() was called, then they start at the - * prefix. With a root set, see DasCat_setRoot(), the mount joins the - * offline catalog instead. + * Paths under sUri load from sUrl and shadow whatever the catalog says + * for them, everything else resolves through the seed as usual. The + * name is in the session's name space: a federated URI, or with a seed + * set, see DasCat_setSeed(), a URI under its prefix or a path from it + * when it is a root, and the mount then joins the offline catalog. May + * be called more than once; when one mount point is under another the + * longer name wins. A mount does not move where relative paths start. * * @param sUrl Where to load the node from, a URL or a file - * @param sUri The catalog URI to mount it at, shorthands accepted + * @param sUri The name to mount it at, shorthands accepted * @returns DAS_OKAY, or DASERR_CAT with the reason logged * @memberof DasCat */ DAS_API DasErrCode DasCat_mount(DasCat* pThis, const char* sUrl, const char* sUri); -/** Number of nodes mounted with DasCat_mount() +/** Number of nodes mounted with DasCat_mount(); the seed is not one * @memberof DasCat */ DAS_API size_t DasCat_mountCount(const DasCat* pThis); -/** True if a root is set, so paths reach only the mounted nodes +/** True if a seed is set, so paths reach only it and the mounted nodes * @memberof DasCat */ -DAS_API bool DasCat_hasRoot(const DasCat* pThis); +DAS_API bool DasCat_hasSeed(const DasCat* pThis); -/** Resolve a path to a node under the root policy +/** Resolve a path to a node * * Network access happens here, one request per catalog level not yet * loaded. Nodes are kept, a second resolve of the same path costs nothing. * * @param sPath A catalog path in any of the forms listed for DasCat. NULL - * or empty means the root itself: the prefix container, the root - * node, or the first mounted node. + * or empty means where relative paths start: the seed, or the + * container the prefix names. * @returns the node, owned by the catalog, or NULL if the path did not * resolve. DasCat_lastUri() then gives the URI that was tried and * DasCat_nearest() the deepest node that did load. @@ -349,7 +353,7 @@ DAS_API DasNode* DasCat_resolve(DasCat* pThis, const char* sPath); /** The full catalog URI of the last path given to DasCat_resolve(), as the * policy understood it, resolved or not. For a path by location this is - * the location. + * the location, and for a root, which has no URI, it is "". * @returns the URI, or NULL before the first resolve * @memberof DasCat */ @@ -357,32 +361,54 @@ DAS_API const char* DasCat_lastUri(const DasCat* pThis); /** Can this URI be reached under the current policy? * - * False when a root is set and no mount covers sUri, and for a local: - * name, see D2URI_LOCAL, that no mount covers; those are never looked up - * in the federated catalog. A failed resolve of a URI this returns false - * for is a policy miss, not a catalog miss, and the message to the user - * should say so. + * False when a seed is set and sUri is neither under its prefix nor + * under a mount. Names below a root seed are paths from it, so under + * one anything with a scheme in front is outside. A failed resolve of + * a URI this returns false for is a policy miss, not a catalog miss, and + * the message to the user should say so. * @memberof DasCat */ DAS_API bool DasCat_covers(const DasCat* pThis, const char* sUri); /** After a failed resolve, the deepest catalog node on the path that did load * - * Strips the last path component and resolves again, repeating until a - * container node loads or the path is exhausted. Costs network requests - * only for levels not already loaded. DasCat_lastUri() afterwards names - * the node returned, so a caller can say "no child X under URI Y" and list - * what Y does have with DasNode_subName(). + * The walk that failed remembers the deepest container it searched and + * the part of the path below it. This loads that container again, which + * costs nothing when it is still in memory and one cache read otherwise. + * DasCat_lastUri() afterwards names the node returned, so a caller can + * say "no child X under URI Y" and list what Y does have with + * DasNode_subName(). * * @param psRest If not NULL, receives a pointer to the part of the failed * path below the node returned, such as "survey/das2", valid until * the next resolve. NULL when nothing is returned. - * @returns the node, owned by the catalog, or NULL if no part of the path - * resolves, the last resolve succeeded, or the path was a location. + * @returns the node, owned by the catalog, or NULL if the last resolve + * succeeded, the path was a location, or no container on the path + * loaded at all, as when no mirror of the das catalog index answered. + * The node may be a root, the seed itself, and DasCat_lastUri() + * is then "". * @memberof DasCat */ DAS_API DasNode* DasCat_nearest(DasCat* pThis, const char** psRest); +/** Load a child of a node, with this session's settings + * + * For a program that has resolved a container, listed its children, see + * DasNode_subName() and dassrc_loadable(), and picked one. The child is + * loaded with the session's credentials, agent and cache, attached under + * pNode, and lives as long as the tree it is in. The same as resolving + * the child's full URI, without composing it. + * + * @param pNode A container node, usually one this session resolved. A + * tree the program loaded itself works too and keeps the child. + * @param sRelPath The path below pNode, "das3" or "survey/das2", with or + * without the separator pNode uses in front. Empty means pNode. + * @returns the node, or NULL after logging why not. DasCat_nearest() + * then says how far the walk got. + * @memberof DasCat + */ +DAS_API DasNode* DasCat_subNode(DasCat* pThis, DasNode* pNode, const char* sRelPath); + /** Resolve a path and build the data source it leads to. * * Resolves the path as DasCat_resolve() does and hands the node to diff --git a/das3/http.c b/das3/http.c index e407afe..3f55501 100644 --- a/das3/http.c +++ b/das3/http.c @@ -383,30 +383,6 @@ char* das_url_toStr(const struct das_url* pUrl, char* sBuf, size_t uLen) return _prn_fit(sBuf, uLen, n); } -size_t das_url_encode(const char* sIn, char* sOut, size_t uOut) -{ - static const char* sHex = "0123456789ABCDEF"; - size_t uNeed = 0; - if(uOut > 0) sOut[0] = '\0'; - for(const unsigned char* p = (const unsigned char*)sIn; p && *p; ++p){ - bool bPlain = isalnum(*p)||(*p == '-')||(*p == '.')||(*p == '_')||(*p == '~'); - size_t n = bPlain ? 1 : 3; - if(uNeed + n < uOut){ - if(bPlain){ - sOut[uNeed] = (char)*p; - } - else{ - sOut[uNeed] = '%'; - sOut[uNeed + 1] = sHex[*p >> 4]; - sOut[uNeed + 2] = sHex[*p & 0x0F]; - } - sOut[uNeed + n] = '\0'; - } - uNeed += n; - } - return uNeed; -} - /* ************************************************************************** */ /* HTTP Client Functions */ @@ -1081,8 +1057,13 @@ static bool _das_http_redirect(DasHttpResp* pRes, DasBuf* pBuf) } void DasHttpResp_freeFields(DasHttpResp* pRes){ - if(pRes->pMime) free(pRes->pMime); - if(pRes->sHeaders) free(pRes->sHeaders); + /* Every string here is heap allocated when set, see the das_string() + calls above and _das_http_setFileName(); a failed request leaves + sError behind and nothing else */ + free(pRes->pMime); pRes->pMime = NULL; + free(pRes->sHeaders); pRes->sHeaders = NULL; + free(pRes->sError); pRes->sError = NULL; + free(pRes->sFilename); pRes->sFilename = NULL; } /* ************************************************************************* */ diff --git a/das3/http.h b/das3/http.h index 6d6cb23..172d7fe 100644 --- a/das3/http.h +++ b/das3/http.h @@ -124,23 +124,6 @@ typedef struct das_http_response_t{ */ DAS_API char* das_url_toStr(const struct das_url* pUrl, char* sBuf, size_t uLen); -/** Percent encode a string for use in a URL query. - * - * Everything but the RFC 3986 unreserved characters, letters, digits and - * "-._~", is encoded, including the ':' in time strings. Servers decode - * before use so over-encoding is harmless, and it is what most encoders - * do. Works like snprintf: the output is always NUL terminated when uOut - * is at least 1, and the return value says how much room was needed. - * - * @param sIn The text to encode. - * @param sOut Receives the encoding. Three times the input length plus - * one always suffices. - * @param uOut The size of sOut. - * @returns The length of the full encoding, not counting the terminator. - * If this is uOut or more the output was cut short. - * @memberof das_url - */ -DAS_API size_t das_url_encode(const char* sIn, char* sOut, size_t uOut); /** Initialize all fields in an http response to default values. diff --git a/das3/node.c b/das3/node.c index aa2623b..1fa9170 100644 --- a/das3/node.c +++ b/das3/node.c @@ -46,7 +46,7 @@ /* Time out values for connections */ -/* Built-in catalog roots. */ +/* The compiled in seed: the index of the das catalog and its mirrors */ static const char* g_lDasDefRoots[2] = { "https://das2.org/catalog/index.json", "https://raw.githubusercontent.com/das-developers/das-cat/master/cat/index.json" @@ -54,7 +54,7 @@ static const char* g_lDasDefRoots[2] = { static int g_nDasDefRoots = 2; -const char** das_root_urls(size_t* pLen){ +const char** das_seed_urls(size_t* pLen){ if(pLen != NULL) *pLen = g_nDasDefRoots; return g_lDasDefRoots; @@ -573,12 +573,11 @@ bool _DasNode_subMatch(DasNode* pBase, const char* sSubPathWithSep) DasNode* new_DasNode_json( const char* pBytes, size_t uBytes, const char* sUrl, const char* sPathUri ){ - /* Every node has a URI; one built from bytes with no name is named by - where the bytes came from, or by a stand in when even that is unknown */ - char sUri[600]; - if(sUrl == NULL) sUrl = "memory:node"; - if((sPathUri == NULL)||(sPathUri[0] == '\0')) _das_uri_local(sUrl, sUri, sizeof(sUri)); - else _das_uri_expand(sPathUri, sUri, sizeof(sUri)); + /* No name given means a root: the node has no URI and what is walked + to below it is named by the path from it */ + char sUri[600] = {'\0'}; + if((sPathUri != NULL) && (sPathUri[0] != '\0')) _das_uri_expand(sPathUri, sUri, sizeof(sUri)); + if(sUrl == NULL) sUrl = ""; DasNode* pNode = _DasNode_fromJson(pBytes, uBytes, sUrl, sUrl, sUri); if(pNode) pNode->bOwner = true; @@ -628,7 +627,7 @@ void del_DasNode(DasNode* pThis) /* ************************************************************************* */ /* URI spellings, see node_priv.h */ -static const char* g_aUriShort[] = {"site:", "test:", "local:"}; +static const char* g_aUriShort[] = {"site:", "test:"}; const char* _das_uri_expand(const char* sIn, char* sOut, size_t uOut) { @@ -654,35 +653,8 @@ const char* _das_uri_expand(const char* sIn, char* sOut, size_t uOut) return sOut; } -const char* _das_uri_local(const char* sLoc, char* sOut, size_t uOut) -{ - const char* sTail = sLoc; - const char* sLead = ""; - if(strncmp(sLoc, "file://", 7) == 0) sTail += 7; - else if(strncmp(sLoc, "http://", 7) == 0){ sTail += 7; sLead = "/"; } - else if(strncmp(sLoc, "https://", 8) == 0){ sTail += 8; sLead = "/"; } - - /* A bare relative path: the working directory is part of the name, - * since "./wav.json" means a different file from every other directory. - * One leading "./" is dropped as pure noise. */ - bool bDrive = isalpha((unsigned char)sTail[0]) && (sTail[1] == ':'); - if((sTail == sLoc) && (sTail[0] != '/') && (sTail[0] != '\\') && !bDrive){ - if((sTail[0] == '.') && ((sTail[1] == '/')||(sTail[1] == '\\'))) sTail += 2; - char sCwd[512] = {'\0'}; - if(getcwd(sCwd, sizeof(sCwd) - 1) == NULL) sCwd[0] = '\0'; - snprintf(sOut, uOut, D2URI_LOCAL "%s%s%s", (sCwd[0] == '/') ? "" : "/", sCwd, DAS_DSEPS); - size_t u = strlen(sOut); - if(u < uOut) snprintf(sOut + u, uOut - u, "%s", sTail); - return sOut; - } - snprintf(sOut, uOut, D2URI_LOCAL "%s%s", (bDrive ? "/" : sLead), sTail); - return sOut; -} - -bool _das_uri_isLocal(const char* sUri) -{ - return (sUri != NULL) && (strncmp(sUri, D2URI_LOCAL, sizeof(D2URI_LOCAL) - 1) == 0); -} +/* ************************************************************************* */ +/* The mount table */ /* Append a JSON string literal, escaping the two characters that matter */ static size_t _jsonStr(char* sOut, size_t uOut, const char* sIn) @@ -698,14 +670,11 @@ static size_t _jsonStr(char* sOut, size_t uOut, const char* sIn) return u; } -/* read comments in node_priv.h header on this function first before edititing it */ -DasNode* _DasNode_newMountRoot( - const char** aUrl, const char** aUri, size_t uMounts, - bool bFallThrough, DasCredMngr* pMgr, const char* sAgent, - const DasNodeCache* pCache -){ +/* read comments in node_priv.h header on this function first before editing it */ +DasNode* _DasNode_newMountTable(const char** aUrl, const char** aUri, size_t uMounts) +{ if(uMounts == 0){ - daslog_error("A mount root needs at least one mount"); + daslog_error("A mount table needs at least one mount"); return NULL; } size_t uTotal = 0; @@ -717,27 +686,23 @@ DasNode* _DasNode_newMountRoot( uTotal += strlen(aUri[i]) + strlen(aUrl[i]); } - /* The mount root is an ordinary Catalog node that only exists in memory. - * Its separator is null, like the federated root's, so child names are - * whole URI prefixes. The mounts come first, then, if asked for, a child - * with the empty name whose URLs are the federated roots. The empty name - * is a prefix of every path, so anything no mount claims falls through - * to the normal catalog, while everything under a mount URI is shadowed. - * Without that child the mounts are all there is. + /* The mount table is an ordinary Catalog node that only exists in memory. + * Its separator is null, like the das catalog index's, so child names + * are whole URI prefixes, and a lookup takes the first child whose name + * is a prefix of the path. * * Sized from the inputs since a mount list has no natural bound. The * escaping below at most doubles a string, hence the factor. */ size_t uJson = 256 + 2 * uTotal + 64 * uMounts; - for(int i = 0; i < g_nDasDefRoots; ++i) uJson += 2 * strlen(g_lDasDefRoots[i]) + 4; char* sJson = (char*)calloc(uJson, 1); if(sJson == NULL){ - das_error(DASERR_NODE, "Couldn't allocate mount root"); + das_error(DASERR_NODE, "Couldn't allocate the mount table"); return NULL; } size_t u = 0; u += snprintf(sJson + u, uJson - u, "{\"type\":\"Catalog\",\"version\":\"0.5\",\"name\":\"mounts\"," - "\"title\":\"Mount root\",\"separator\":null,\"catalog\":{" + "\"title\":\"Mount table\",\"separator\":null,\"catalog\":{" ); /* Longest URI first, see node_priv.h. A selection sort over an index @@ -764,14 +729,6 @@ DasNode* _DasNode_newMountRoot( u += _jsonStr(sJson + u, uJson - u, aUrl[k]); u += snprintf(sJson + u, uJson - u, "]}"); } - if(bFallThrough){ - u += snprintf(sJson + u, uJson - u, ",\"\":{\"type\":\"Catalog\",\"urls\":["); - for(int i = 0; i < g_nDasDefRoots; ++i){ - if(i > 0) u += snprintf(sJson + u, uJson - u, ","); - u += _jsonStr(sJson + u, uJson - u, g_lDasDefRoots[i]); - } - u += snprintf(sJson + u, uJson - u, "]}"); - } u += snprintf(sJson + u, uJson - u, "}}"); if(u >= uJson - 1){ daslog_error("Mount URI or URL too long"); @@ -779,16 +736,471 @@ DasNode* _DasNode_newMountRoot( return NULL; } - /* Nothing is fetched here, the first subNode call does that and takes - the manager, agent and cache from the root */ - DasNode* pRoot = _DasNode_fromJson(sJson, u, "memory:mounts", "memory:mounts", ""); + /* Nothing is fetched here, the first walk through the table does that */ + DasNode* pTable = _DasNode_fromJson(sJson, u, "memory:mounts", "memory:mounts", ""); free(sJson); - if(pRoot != NULL){ - pRoot->bOwner = true; - pRoot->pMgr = pMgr; - pRoot->sAgent = sAgent; - pRoot->pCache = pCache; + if(pTable != NULL) pTable->bOwner = true; + return pTable; +} + +/* ************************************************************************* */ +/* Walking: resolving a path below a node, one document at a time + * + * The walk is the old recursive loader turned into an explicit stack, one + * frame per path level, with the fetch taken out. A frame searches one + * container for the child that starts the rest of the path, first among + * the children already loaded under it, then in its listing. A listing + * entry names one or more URLs; they are the candidates of a fetch step, + * offered in catalog order. A document that arrives is parsed and either + * finishes the walk, becomes the parent of the next frame, or is dropped + * because nothing under it could continue the path, in which case the + * next candidate is offered. When a frame runs out of candidates and + * children the walk backs out to the frame above, which drops the node it + * descended into and moves to its next candidate. + * + * First match wins at every level, as it always has: for the path "ab/x" + * a child named "a" is tried before "ab", and the separator check in + * _DasNode_subMatch() is what sends the walk on to "ab". A listing entry + * whose name is already among the loaded children is skipped, since the + * loaded copy was tried first and a second copy would only duplicate it. + * + * Nothing attaches to the tree until the walk succeeds. Then every node + * it loaded joins its parent's child list and the walk forgets them. + * + * The seed walk, new_DasNodeWalk_seed(), has one frame with no parent + * whose candidates are the compiled in seed URLs. The node it returns is + * a tree top the caller owns, and lookups by full URI are walks from it; + * a session keeps it for its lifetime, see catalog.c. */ + +#define _WALK_DEPTH 64 /* more levels than any catalog path has */ +#define _WALK_URI_SZ 600 + +struct walk_frame { + DasNode* pParent; /* container searched at this level, NULL for a seed */ + const char* sRest; /* path below pParent, its separator stripped */ + bool bListing; /* loaded children exhausted, now in the listing */ + ptrdiff_t iLoaded; /* next loaded child to look at */ + const das_json_dict_el* pChild; /* listing entry in hand, NULL before the first */ + const char* sChild; /* its name, borrowed from the parent's document */ + const DasJdo* pUrls; /* its urls array, NULL for a seed */ + size_t iUrl; /* first candidate still on offer */ + size_t uUrls; /* candidates in all, string entries only */ + DasNode* pNode; /* loaded for candidate iUrl, not yet attached */ + char sUri[_WALK_URI_SZ]; /* the URI of the node this frame loads */ +}; + +struct das_node_walk { + char* sPath; /* owned copy, frames point into it */ + DasNode* pFrom; /* borrowed, NULL when loading a seed */ + const char** aSeedUrl; + size_t uSeedUrl; + struct walk_frame aFrame[_WALK_DEPTH]; + size_t uFrames; + das_walk_state_e state; + DasNode* pResult; + char sNear[_WALK_URI_SZ]; /* deepest container searched, "" if none */ + const char* sNearRest; + size_t uDeepest; +}; + +static void _DasNode_attach(DasNode* pParent, DasNode* pChild, const char* sName) +{ + DasCatNode* pCat = (DasCatNode*)pParent; + DasAry_append(pCat->pSubNodes, (const ubyte*)&pChild, 1); + DasAry_append(pCat->pSubPaths, (const ubyte*)sName, strlen(sName) + 1); + DasAry_markEnd(pCat->pSubPaths, DIM1); +} + +/* The name a loaded child was attached under, "" for a bad entry */ +static const char* _DasCatNode_loadedName(const DasCatNode* pCat, ptrdiff_t i, size_t* pLen) +{ + size_t uLen = 0; + const char* s = (const char*)DasAry_getBytesIn(pCat->pSubPaths, DIM1_AT(i), &uLen); + if((s == NULL)||(uLen == 0)){ *pLen = 0; return ""; } + *pLen = uLen - 1; /* the stored NUL is not part of the name */ + return s; +} + +static bool _DasCatNode_hasLoaded(const DasCatNode* pCat, const char* sName) +{ + ptrdiff_t n = DasAry_lengthIn(pCat->pSubNodes, DIM0); + for(ptrdiff_t i = 0; i < n; ++i){ + size_t uLen = 0; + const char* s = _DasCatNode_loadedName(pCat, i, &uLen); + if((uLen == strlen(sName)) && (strncmp(s, sName, uLen) == 0)) return true; } - return pRoot; + return false; } +/* Candidate number n of a frame, counting string entries only */ +static const char* _walk_url(const DasNodeWalk* pThis, const struct walk_frame* pF, size_t n) +{ + if(pF->pUrls == NULL) return (n < pThis->uSeedUrl) ? pThis->aSeedUrl[n] : NULL; + size_t k = 0; + for(const das_json_ary_el* pEl = DasJdo_aryFirst(pF->pUrls); pEl != NULL; pEl = pEl->next){ + const char* s = DasJdo_string(pEl->value); + if(s == NULL) continue; + if(k++ == n) return s; + } + return NULL; +} + +static size_t _walk_countUrls(const DasNode* pParent, const char* sChild, const DasJdo* pUrls) +{ + size_t u = 0; + int i = 0; + for(const das_json_ary_el* pEl = DasJdo_aryFirst(pUrls); pEl != NULL; pEl = pEl->next, ++i){ + if(DasJdo_string(pEl->value) != NULL) ++u; + else daslog_error_v("From %s: %s/%s/%d element is not a string", + pParent->sURL, D2FRAG_URLS, sChild, i); + } + return u; +} + +/* Remember the deepest container on the path, for DasNodeWalk_nearest(). + * On the path means the rest of the path continues below it with its + * separator, the same test _DasNode_subMatch() starts with; a look-alike + * such as "a" for "ab/x" is not on the path. Whether anything under it + * matches does not matter: "uiowa" is the nearest node for + * "uiowa/nope/deeper" even though it lists no "nope". */ +static void _walk_nearSet(DasNodeWalk* pThis, size_t uDepth, const DasNode* pNode, const char* sRest) +{ + if(uDepth <= pThis->uDeepest) return; + pThis->uDeepest = uDepth; + strncpy(pThis->sNear, pNode->sPath, sizeof(pThis->sNear) - 1); + pThis->sNearRest = sRest; +} + +/* For a child met on the way: it is on the path only if the rest continues + * below it with its separator */ +static void _walk_near(DasNodeWalk* pThis, size_t uDepth, const DasNode* pNode, const char* sRem) +{ + const char* sSep = ((const DasCatNode*)pNode)->sPathSep; + size_t uSep = strlen(sSep); + if(uSep > 0){ + if(strncmp(sRem, sSep, uSep) != 0) return; + sRem += uSep; + } + _walk_nearSet(pThis, uDepth, pNode, sRem); +} + +static bool _walk_push(DasNodeWalk* pThis, DasNode* pParent, const char* sRest) +{ + if(pThis->uFrames == _WALK_DEPTH){ + daslog_error_v("Catalog path %s is more than %d levels deep", pThis->sPath, _WALK_DEPTH); + return false; + } + struct walk_frame* pF = pThis->aFrame + pThis->uFrames; + memset(pF, 0, sizeof(*pF)); + pF->pParent = pParent; + + /* The rest may start with the separator this container uses, "/uiowa" + under site: or not, "uiowa"; both name the same child */ + const char* sSep = ((DasCatNode*)pParent)->sPathSep; + size_t uSep = strlen(sSep); + if((uSep > 0) && (strncmp(sRest, sSep, uSep) == 0)) sRest += uSep; + pF->sRest = sRest; + pThis->uFrames += 1; + _walk_nearSet(pThis, pThis->uFrames, pParent, sRest); /* already stripped */ + return true; +} + +/* Back out one level. The frame above drops the node it descended into + and moves to its next candidate; a frame that descended into an already + loaded child has nothing to drop. */ +static void _walk_pop(DasNodeWalk* pThis) +{ + struct walk_frame* pF = pThis->aFrame + (pThis->uFrames - 1); + if(pF->pNode != NULL){ _DasNode_delTree(pF->pNode); pF->pNode = NULL; } + pThis->uFrames -= 1; + if(pThis->uFrames == 0) return; + pF = pThis->aFrame + (pThis->uFrames - 1); + if(pF->pNode != NULL){ + _DasNode_delTree(pF->pNode); + pF->pNode = NULL; + pF->iUrl += 1; + } +} + +static void _walk_finish(DasNodeWalk* pThis, DasNode* pResult) +{ + for(size_t k = 0; k < pThis->uFrames; ++k){ + struct walk_frame* pF = pThis->aFrame + k; + if(pF->pNode == NULL) continue; + if(pF->pParent != NULL) _DasNode_attach(pF->pParent, pF->pNode, pF->sChild); + pF->pNode = NULL; + } + if(pThis->pFrom == NULL) pResult->bOwner = true; /* a seed, the caller's */ + pThis->pResult = pResult; + pThis->state = das_walk_done; +} + +/* Move to the next state: a fetch step, done, or failed */ +static void _walk_advance(DasNodeWalk* pThis) +{ + while(pThis->uFrames > 0){ + struct walk_frame* pF = pThis->aFrame + (pThis->uFrames - 1); + + /* Candidates still on offer for the child in hand */ + if((pF->pNode == NULL) && pF->bListing && (pF->iUrl < pF->uUrls)){ + pThis->state = das_walk_fetch; + return; + } + if(pF->pParent == NULL){ _walk_pop(pThis); continue; } /* every seed URL tried */ + + DasCatNode* pCat = (DasCatNode*)pF->pParent; + + /* Children already loaded, so a walked path costs no document */ + if(!pF->bListing){ + ptrdiff_t n = DasAry_lengthIn(pCat->pSubNodes, DIM0); + bool bPushed = false; + while(pF->iLoaded < n){ + ptrdiff_t i = pF->iLoaded++; + size_t uLen = 0; + const char* sName = _DasCatNode_loadedName(pCat, i, &uLen); + if(uLen == 0){ + daslog_warn_v("Node at %s has a zero-length sub-path", pF->pParent->sURL); + continue; + } + if(strncmp(sName, pF->sRest, uLen) != 0) continue; + DasNode* pKid = *((DasNode**)DasAry_getAt(pCat->pSubNodes, vtUnknown, IDX0(i))); + const char* sRem = pF->sRest + uLen; + if(sRem[0] == '\0'){ _walk_finish(pThis, pKid); return; } + if(DasNode_isCatalog(pKid)) _walk_near(pThis, pThis->uFrames + 1, pKid, sRem); + if(!_DasNode_subMatch(pKid, sRem)) continue; + if(_walk_push(pThis, pKid, sRem)){ bPushed = true; break; } + } + if(bPushed) continue; + pF->bListing = true; + pF->pChild = NULL; + } + + /* The listing: the next child whose name starts the rest */ + const DasJdo* pDir = (const DasJdo*)pCat->pContainer; + const das_json_dict_el* pEl = NULL; + if((pDir != NULL) && (pDir->type == das_json_type_dict)) + pEl = (pF->pChild == NULL) ? DasJdo_dictFirst(pDir) : pF->pChild->next; + else + daslog_error_v("Catalog container node missing for node from %s", pF->pParent->sURL); + + bool bOffer = false; + for(; pEl != NULL; pEl = pEl->next){ + const char* sChild = pEl->name->string; + size_t uLen = strlen(sChild); + if(strncmp(sChild, pF->sRest, uLen) != 0) continue; + if(_DasCatNode_hasLoaded(pCat, sChild)) continue; + + /* Broken syntax: stop using this listing, the level above may + have another copy */ + const DasJdo* pStub = pEl->value; + if((pStub == NULL)||(pStub->type != das_json_type_dict)){ + daslog_error_v("Catalog error @ %s: sub item '%s' is not a dictionary", + pF->pParent->sURL, sChild); + break; + } + const DasJdo* pUrls = DasJdo_get(pStub, D2FRAG_URLS); + if((pUrls == NULL)||(pUrls->type != das_json_type_ary)){ + daslog_error_v("From %s: %s element of node '%s' doesn't have a %s array", + pF->pParent->sURL, pCat->sContainer, sChild, D2FRAG_URLS); + break; + } + pF->pChild = pEl; + pF->sChild = sChild; + pF->pUrls = pUrls; + pF->iUrl = 0; + pF->uUrls = _walk_countUrls(pF->pParent, sChild, pUrls); + /* A root has no URI, so its children are named by bare name and + the levels below by their path from the root */ + if(pF->pParent->sPath[0] == '\0') + snprintf(pF->sUri, sizeof(pF->sUri), "%s", sChild); + else + snprintf(pF->sUri, sizeof(pF->sUri), "%s%s%s", pF->pParent->sPath, pCat->sPathSep, sChild); + if(pF->uUrls == 0) continue; + bOffer = true; + break; + } + if(bOffer){ pThis->state = das_walk_fetch; return; } + + /* A miss at one level is normal while the walk backs out and tries + other candidates or children, so it is not an error here. The + driver knows whether the whole path failed and has the user's + wording of it. */ + const char* sLabel = DasNode_label(pF->pParent); + daslog_info_v("Node %s (URI '%s') has no child node that starts with %s", + sLabel ? sLabel : "", pF->pParent->sPath, pF->sRest); + _walk_pop(pThis); + } + pThis->state = das_walk_failed; +} + +static DasNodeWalk* _walk_alloc(DasNode* pFrom, const char* sRelPath) +{ + DasNodeWalk* pThis = (DasNodeWalk*)calloc(1, sizeof(DasNodeWalk)); + if(pThis == NULL){ + das_error(DASERR_NODE, "Couldn't allocate a catalog walk"); + return NULL; + } + pThis->pFrom = pFrom; + + /* An owned copy even when empty, das_strdup() has nothing to give for "" */ + size_t uPath = (sRelPath != NULL) ? strlen(sRelPath) : 0; + pThis->sPath = (char*)calloc(uPath + 1, 1); + if(pThis->sPath == NULL){ + das_error(DASERR_NODE, "Couldn't allocate a catalog walk"); + free(pThis); + return NULL; + } + if(uPath > 0) memcpy(pThis->sPath, sRelPath, uPath); + return pThis; +} + +DasNodeWalk* new_DasNodeWalk(DasNode* pFrom, const char* sRelPath) +{ + if(pFrom == NULL){ + daslog_error("A walk needs a node to start from; for the compiled in seed " + "see new_DasNodeWalk_seed()"); + return NULL; + } + if(!DasNode_isCatalog(pFrom)){ + daslog_error_v("Node %s from %s is a terminating node", + DasNode_label(pFrom), DasNode_srcUrl(pFrom)); + return NULL; + } + DasNodeWalk* pThis = _walk_alloc(pFrom, sRelPath); + if(pThis == NULL) return NULL; + + if(pThis->sPath[0] == '\0'){ + pThis->pResult = pFrom; + pThis->state = das_walk_done; + return pThis; + } + _walk_push(pThis, pFrom, pThis->sPath); /* cannot be too deep at 0 */ + _walk_advance(pThis); + return pThis; +} + +DasNodeWalk* new_DasNodeWalk_seed(void) +{ + DasNodeWalk* pThis = _walk_alloc(NULL, NULL); + if(pThis == NULL) return NULL; + pThis->aSeedUrl = das_seed_urls(&pThis->uSeedUrl); + + /* One frame with no parent: its candidates are the seed URLs and the + document that arrives is the result */ + struct walk_frame* pF = pThis->aFrame; + pF->sRest = pThis->sPath; + pF->bListing = true; + pF->uUrls = pThis->uSeedUrl; + pThis->uFrames = 1; + _walk_advance(pThis); + return pThis; +} + +void del_DasNodeWalk(DasNodeWalk* pThis) +{ + if(pThis == NULL) return; + /* Frames hold unattached nodes only while the walk is under way; each + is a separate tree since the next frame's node is not attached to + it, so order does not matter */ + for(size_t k = 0; k < pThis->uFrames; ++k) + if(pThis->aFrame[k].pNode != NULL) _DasNode_delTree(pThis->aFrame[k].pNode); + free(pThis->sPath); + free(pThis); +} + +das_walk_state_e DasNodeWalk_state(const DasNodeWalk* pThis){ return pThis->state; } + +size_t DasNodeWalk_urlCount(const DasNodeWalk* pThis) +{ + if(pThis->state != das_walk_fetch) return 0; + const struct walk_frame* pF = pThis->aFrame + (pThis->uFrames - 1); + return pF->uUrls - pF->iUrl; +} + +const char* DasNodeWalk_url(const DasNodeWalk* pThis, size_t uIdx) +{ + if(uIdx >= DasNodeWalk_urlCount(pThis)) return NULL; + const struct walk_frame* pF = pThis->aFrame + (pThis->uFrames - 1); + return _walk_url(pThis, pF, pF->iUrl + uIdx); +} + +const char* DasNodeWalk_uri(const DasNodeWalk* pThis) +{ + if(pThis->state != das_walk_fetch) return ""; + return pThis->aFrame[pThis->uFrames - 1].sUri; +} + +DasErrCode DasNodeWalk_feed( + DasNodeWalk* pThis, size_t uIdx, const char* pBytes, size_t uBytes, + const char* sFinalUrl +){ + /* Misuse by the driver, logged rather than raised: a host that mixes up + its candidates should not take the program down with it */ + if(pThis->state != das_walk_fetch){ + daslog_error("The walk is not waiting for a document"); + return DASERR_NODE; + } + size_t uOffer = DasNodeWalk_urlCount(pThis); + if(uIdx >= uOffer){ + daslog_error_v("Candidate %zu is past the %zu on offer", uIdx, uOffer); + return DASERR_NODE; + } + + struct walk_frame* pF = pThis->aFrame + (pThis->uFrames - 1); + const char* sUrl = _walk_url(pThis, pF, pF->iUrl + uIdx); + if(sFinalUrl == NULL) sFinalUrl = sUrl; + + DasNode* pNode = _DasNode_fromJson(pBytes, uBytes, sUrl, sFinalUrl, pF->sUri); + if((pNode != NULL) && (pF->pParent == NULL) && (pNode->nType != d2node_catalog)){ + daslog_info_v("%s is not a Catalog node, skipping it as a seed", sUrl); + _DasNode_delTree(pNode); + pNode = NULL; + } + if(pNode == NULL){ + pF->iUrl += uIdx + 1; + _walk_advance(pThis); + return DASERR_NODE; + } + pF->iUrl += uIdx; /* the one in hand; a back out moves past it */ + pF->pNode = pNode; + + const char* sRem = (pF->pParent == NULL) ? pF->sRest : pF->sRest + strlen(pF->sChild); + if(sRem[0] == '\0'){ _walk_finish(pThis, pNode); return DAS_OKAY; } + + if(DasNode_isCatalog(pNode)) _walk_near(pThis, pThis->uFrames + 1, pNode, sRem); + if(DasNode_isCatalog(pNode) && _DasNode_subMatch(pNode, sRem) && _walk_push(pThis, pNode, sRem)){ + _walk_advance(pThis); + return DAS_OKAY; + } + + /* A good document that leads nowhere: a look-alike child, or a mirror + whose listing is behind. Not the driver's problem. */ + daslog_debug_v("Nothing under %s continues the path %s", sUrl, sRem); + _DasNode_delTree(pNode); + pF->pNode = NULL; + pF->iUrl += 1; + _walk_advance(pThis); + return DAS_OKAY; +} + +void DasNodeWalk_fail(DasNodeWalk* pThis, size_t uIdx, const char* sWhy) +{ + if(uIdx >= DasNodeWalk_urlCount(pThis)) return; + struct walk_frame* pF = pThis->aFrame + (pThis->uFrames - 1); + daslog_debug_v("%s: %s", _walk_url(pThis, pF, pF->iUrl + uIdx), sWhy ? sWhy : "not available"); + pF->iUrl += uIdx + 1; + _walk_advance(pThis); +} + +DasNode* DasNodeWalk_result(const DasNodeWalk* pThis) +{ + return (pThis->state == das_walk_done) ? pThis->pResult : NULL; +} + +const char* DasNodeWalk_nearest(const DasNodeWalk* pThis, const char** psRest) +{ + if(psRest != NULL) *psRest = NULL; + if((pThis->state != das_walk_failed)||(pThis->uDeepest == 0)) return NULL; + if(psRest != NULL) *psRest = pThis->sNearRest; + return pThis->sNear; +} diff --git a/das3/node.h b/das3/node.h index 268423e..1da4bb0 100644 --- a/das3/node.h +++ b/das3/node.h @@ -30,97 +30,47 @@ extern "C" { /** @addtogroup catalog * - * @par Working with nodes directly - * - * The node constructors in this header are what DasCat is built on. Use - * them when a program manages its own trees, for instance to hold two - * unrelated catalogs at once, or to load a single isolated node from a URL - * and nothing else. - * - * There are two ways to get a node. new_DasNode() finds one by its path - * URI from the top of the federated das2 catalog, for example - * tag:das2.org,2012:site:/swri/mars_express/aspera3/sciels, walking down - * from the global root, which is the the un-named node at: - * - * http://das2.org/catalog/index.json - * - * and its mirrors. new_DasNode_url() loads one from an explicit URL or - * file, for example: - * - * http://das2.org/catalog/das/site/uiowa/cassini/rpws/survey.json - * - * and does not walk down from the global catalog root at all. + * @par Nodes without a session * - * Ownership follows general das2C rules: the node a constructor - * (new_DasNode, new_DasNode_url) hands you is yours, and del_DasNode() on - * it frees the whole tree loaded below it. Children fetched with - * DasNode_subNode() are managed by their parent and go with it; - * del_DasNode() ignores them. new_DasNode() discards the nodes it walked - * through on the way down, so what you hold is a tree with no parent in - * memory, whether or not it is the top of any catalog. + * A DasNode is one parsed catalog document: its type, URI, source URL, + * and the document itself, with its children listed by name and type and + * the URLs each would load from. Nothing in this header fetches. A node + * is built from bytes the caller obtained, see new_DasNode_json(), and a + * path below it is resolved by a DasNodeWalk, which says which document + * it needs next and takes what the caller hands back. Programs on a real + * machine do not drive the walk themselves: a DasCat session, see + * catalog.h, reads files, keeps a cache, makes the HTTP requests and + * loops the walk until it stops. A browser build drives it from + * JavaScript with fetch(). The compiled in seed, the index of the das2 + * catalog, is loaded through the same walk interface by + * new_DasNodeWalk_seed(). * - * Catalog nodes do not know their own names. Like files on a disk, a - * node's URI comes from the entries above it, up to the global root, which - * has none. So a node loaded from a URL is named by the caller, or failing - * that, by its location under D2URI_LOCAL. + * Ownership follows general das2C rules: the node a constructor hands + * you is yours, and del_DasNode() on it frees the whole tree loaded below + * it. Children a walk attaches are managed by their parent and go with + * it; del_DasNode() ignores them. * - * An example of getting a data source interface definition follows: - * @code{.cpp} - - // Load the U. Iowa site catalog. The nodes walked through to reach it - // are freed before this returns. - - DasNode* pSite = new_DasNode("tag:das2.org,2012:site:/uiowa", NULL, NULL); - if(!DasNode_isCatalog(pSite)) handle_error(); - - // Use the site catalog to get the Cassini Saturn SLS2 data source description - - const char* sRelPath = "cassini/ephemeris/saturn_sls2"; - DasNode* pNode = DasNode_subNode(pSite, sRelPath, NULL, NULL); - - if(!DasNode_isStreamSrc(pNode)) handle_error(); - - // Get the HTTP GET protocol definition - - const DasJdo* pDef = DasNode_getJdo(pNode, "protocol"); - if(pDef == NULL) handle_error(); - - // Freeing the site node frees everything loaded below it, pNode and the - // document pDef points into included - - del_DasNode(pSite); - - @endcode + * Catalog nodes do not know their names. Like files on a disk, a + * node's URI comes from the entries above it, up to the root, which has + * none. So a node built from bytes is named by the caller, or not at + * all: a seed given no prefix is a root, and the nodes below it are named + * by their path from it. * - * You now have a JSON object describing how the source is queried. The - * layout of HttpStreamSrc nodes is given by the das catalog schema, see - * DasNode_version() for the two layouts in the wild. + * Driving a walk by hand, which is what a browser build does: * - * For isolated sites, or for testing purposes catalog nodes can be loaded - * directly. The following example acquires a local test catalog and then a - * specific test node. - * - @code - - // Assume this site uses a password since it is a testing area. Create a - // credentials manager and pass a basic auth hash so we aren't bothered during - // testing. - - DasCredMngr* pMngr = new_CredMngr(NULL); - das_credential cred; - das_cred_init(&cred, "my.domain.com", "Regression Test Catalog", NULL, - "VVNFUk5BTUU6UEFTU1dPUkQ="); - CredMngr_addCred(pMngr, &cred); - - const char* sTestUrl = "http://my.domain.com/das2/regtest.json"; - DasNode* pTop = new_DasNode_url(sTestUrl, "regtest", pMngr, NULL); - if(!DasNode_isCatalog(pTop)) handle_error(); - - // At this point sub-items can be loaded as above. The credentials manager - // given to the top node is inherited by the lookup. - - DasNode* pNode = DasNode_subNode(pTop, "MarsExpress/aspera/els", NULL, NULL); - + * @code + DasNode* pSite = new_DasNode_json(sBytes, uLen, sUrl, "tag:das2.org,2012:site:/uiowa"); + + DasNodeWalk* pWalk = new_DasNodeWalk(pSite, "juno/wav/survey"); + while(DasNodeWalk_state(pWalk) == das_walk_fetch){ + const char* sNext = DasNodeWalk_url(pWalk, 0); + if(my_fetch(sNext, &sDoc, &uDoc)) + DasNodeWalk_feed(pWalk, 0, sDoc, uDoc, NULL); + else + DasNodeWalk_fail(pWalk, 0, "no answer"); + } + DasNode* pSurvey = DasNodeWalk_result(pWalk); // NULL if the walk failed + del_DasNodeWalk(pWalk); // pSurvey lives on under pSite @endcode */ @@ -129,14 +79,6 @@ extern "C" { #define D2URI_SITES D2URI_ROOT "site" #define D2URI_SITE_UIOWA D2URI_SITES ":/uiowa" -/* Names for nodes that aren't part of the federated catalog system. - * A file or URL loaded with no designated URI gets - * "tag:das2.org,2012:local:" followed by its location. Like a - * private IP range the space is reserved by the machinery, not listed - * anywhere: nothing under it is ever looked up through the federated - * roots. */ -#define D2URI_LOCAL D2URI_ROOT "local:" - /* Defines for common document fragments */ #define D2FRAG_TYPE "type" #define D2FRAG_VERSION "version" @@ -171,125 +113,25 @@ typedef enum das_node_type_enum { /** A das catalog node, opaque. * - * Nodes are made by the new_DasNode*() constructors and DasNode_subNode(), - * read through the accessors below, and freed by del_DasNode() on the - * node a constructor returned, which takes its sub-nodes with it. + * Nodes are made by new_DasNode_json() and by a DasNodeWalk, read + * through the accessors below, and freed by del_DasNode() on a tree top, + * which takes the nodes walked to below it along. */ typedef struct das_node DasNode; /** @} */ -/** Return the compiled in catalog root URLs - * @param pLen - A pointer to a size_t to receive the number of builtin roots - * @return A pointer to an array of null terminated strings. - */ -DAS_API const char** das_root_urls(size_t* pLen); - -/** Load a node by its catalog URI - * - * Get a catalog node that is not attached to any parent nodes. If the - * node acquired is a container, see DasNode_isCatalog(), then it can be - * used to acquire further nodes. - * - * This function consults the distributed das2 catalog to find and load nodes. - * See new_DasNode_url() for a version that only loads a specified URL. - * - * @code{.cpp} - - DasNode* pRoot = NULL; - - // Get the das2 site root catalog node - pRoot = new_DasNode(D2URI_SITES, NULL, NULL); - - // Get the U. Iowa site catalog top node - pRoot = new_DasNode(D2URI_SITE_UIOWA, NULL, NULL); - - @endcode - * - * @param sPathUri Retrieve a catalog node given a global path URI. This a - * location from the top of the federated catalog system, not - * a file system or web path. NULL loads the federated root - * itself. The shorthands "site:/..." and "test:/..." - * are accepted for the two well known children of the das2 - * root. - * - * @param pMgr A credentials manager object to consult if a password is - * requested. May be set to NULL to indicate that only public - * items may be requested. Usually this is not needed as most - * catalog entries are public though the data sources they - * describe may not be. The returned node remembers it and - * hands it to anything loaded through DasNode_subNode(). - * - * @param sAgent The user agent string you wish to send to the server. If - * NULL then the string "libdas/3" is sent. Borrowed, not - * copied, so a string literal or a buffer that outlives the - * node. - * - * @return A new DasNode object allocated on the heap, or NULL if object - * resolution failed. Calling del_DasNode() for the returned - * pointer will delete all sub-nodes from memory as well. - * - * @memberof DasNode - */ -DAS_API DasNode* new_DasNode( - const char* sPathUri, DasCredMngr* pMgr, const char* sAgent -); - -/** Load a node from a URL or file - * - * Get a catalog node that is not attached to any parent nodes. If the - * node acquired is a container, see DasNode_isCatalog(), then it can be - * used to acquire further nodes. - * - * There are two basic methods of node acquisition, set a path URI and let the - * library find the node using the built-in global catalog location, or - * provide an explicit URL. - * - * @code{.cpp} - - DasNode* pRoot = NULL; - - // Get a standalone local Voyager catalog root - pRoot = new_DasNode_url("http://my.site.gov/test/local_vgr.json", "vgr", NULL, NULL); - - // Get a standalone data source description for Voyager PLS data - pRoot = new_DasNode_url("http://my.site.gov/test/local_vgr/pls.json", "vgr/pls", NULL, NULL); - - // Nodes need not come from a server, a path or file:// URL reads a file - pRoot = new_DasNode_url("./local_vgr.json", "vgr", NULL, NULL); - - @endcode - * - * @param sUrl Retrieve a catalog node from an explicit URL. No searches - * will be performed. Anything that is not an http(s) URL is - * read as a local file, with or without a "file://" prefix. - * - * @param sPathUri Since the node was not loaded by following the federated - * catalog system to an end point, you have to tell the node - * its name. Any name works, it is saved and reported by - * DasNode_pathUri(), and children get it as a prefix. The - * shorthands "site:", "test:" and "local:" are expanded. - * NULL or empty names the node after its location under - * D2URI_LOCAL, so every node has a URI; give the real - * catalog URI instead when the file stands in for a - * catalog node, see DasCat_mount(). - * - * @param pMgr A credentials manager object to consult if a password is - * requested. May be set to NULL to indicate that only public - * items may be requested. Inherited by DasNode_subNode(). - * - * @param sAgent The user agent string you wish to send to the server. If - * NULL then the string "libdas/3" is sent. Borrowed, not - * copied. - * - * @return A new DasNode object allocated on the heap, or NULL if object - * resolution failed. Use DasNode_type() to determine which - * type of catalog item was acquired. - * - * @memberof DasNode +/** The compiled in seed URLs: the index of the das catalog and its mirrors + * + * The seed is a root, a node with no name, whose children are the top + * level catalogs, each named by a whole URI prefix such as + * "tag:das2.org,2012:". new_DasNodeWalk_seed() loads it from whichever + * of these answers, in order. + * + * @param pLen Receives the number of URLs. May be NULL. + * @return An array of NUL terminated strings, static. */ -DAS_API DasNode* new_DasNode_url(const char* sUrl, const char* sPathUri, - DasCredMngr* pMgr, const char* sAgent); +DAS_API const char** das_seed_urls(size_t* pLen); /** Build a node from the JSON text of a node document * @@ -304,7 +146,8 @@ DAS_API DasNode* new_DasNode_url(const char* sUrl, const char* sPathUri, * @param sUrl Where the text came from, recorded as the node's source * URL, see DasNode_srcUrl(). May be NULL. * @param sPathUri The catalog URI of the node, shorthands expanded. NULL - * or empty names the node after sUrl under D2URI_LOCAL. + * or empty makes it a root: no URI, and the nodes walked to + * below it are named by their path from it. * @return A new node, to be freed with del_DasNode(), or NULL after * logging a parse problem or an unrecognized node type. * @memberof DasNode @@ -313,39 +156,6 @@ DAS_API DasNode* new_DasNode_json( const char* pBytes, size_t uBytes, const char* sUrl, const char* sPathUri ); -/** Get a das2 catalog node contained item. - * - * This function will search down the catalog hierarchy and find the requested - * node, downloading any intermediate nodes as needed. If the descendant node - * has already been downloaded no new network activity is generated. - * - * @param pThis A container node, one for which DasNode_isCatalog() is true - * - * @param sRelPath the ID of the descendant node. This id is the portion - * that would be appended to this node's PathUri to make the - * complete URI of the final node. sRelPath does not need to start - * with the sub-item separator (if one is defined for this catalog) - * but may at the callers option. For example, both: - * ":/uiowa" and "uiowa" work for getting the uiowa subnode from - * the site catalog, but "/uiowa" would not since the separator - * defined for site is ":/". - * - * @param pMgr A credentials manager object to consult if a password is - * requested. NULL means inherit the manager this node was - * loaded with, which is the one given to new_DasNode() or - * its relatives. Pass one to override it for this lookup. - * - * @param sAgent The user agent string to send. NULL inherits the agent this - * node was loaded with; if none was ever given, "libdas/3". - * - * @return A pointer to the child node, or NULL if no such child existed - * in the node cache nor could not be downloaded. - * @memberof DasNode - */ -DAS_API DasNode* DasNode_subNode( - DasNode* pThis, const char* sRelPath, DasCredMngr* pMgr, const char* sAgent -); - /** Returns true this node can contain sub nodes. * @memberof DasNode */ @@ -367,27 +177,33 @@ DAS_API bool DasNode_isSpaseRec(const DasNode* pNode); DAS_API bool DasNode_isSpdfCat(const DasNode* pNode); /** Get the path URI for this catalog node - * @param pThis the catalog node - * @return A pointer to the path URI string. All catalog nodes have a path URI + * + * Names are composed walking down: a seed given a prefix hands it to its + * children with its separator between, and so on. A seed given no + * prefix is a root, and a root is the one node with no URI; the compiled + * in das catalog index is the usual example. + * + * @param pThis the catalog node + * @return The URI, or NULL for a root * @memberof DasNode */ DAS_API const char* DasNode_pathUri(const DasNode* pThis); -/** Get the location from which this catalog node was read. - * @param pThis the catalog node - * @return A pointer to the source URL string. All catalog nodes are loaded - * from somewhere, das2C does not provide functions to generate - * node objects programmatically. +/** Get the location from which this catalog node was read + * @param pThis the catalog node + * @return The source URL, or "" for a node built from bytes with no + * location given, see new_DasNode_json(). Never NULL. * @memberof DasNode */ DAS_API const char* DasNode_srcUrl(const DasNode* pThis); /** Get the type of node - * This is a more specific question than the 'is' functions below, which should - * probably be used instead so that application code can work in a "duck-typing" - * manner. - * @param pThis the catalog node - * @return The node type from the + * + * A more specific question than the 'is' functions above, which are + * usually the better choice so that application code works in a duck + * typing manner. + * @param pThis the catalog node + * @return The node type, see das_node_type_e * @memberof DasNode */ DAS_API das_node_type_e DasNode_type(const DasNode* pThis); @@ -431,7 +247,7 @@ DAS_API const char* DasNode_version(const DasNode* pThis); */ DAS_API const char* DasNode_label(const DasNode* pThis); -/** Get the separator a container node puts between its own path URI and +/** Get the separator a container node puts between its path URI and * the name of a child. * * @returns "/" by default, the empty string if the node's "separator" is @@ -462,7 +278,7 @@ DAS_API const char* DasNode_subName(const DasNode* pThis, size_t uIdx); * * The stub is the dictionary the container holds for the child. It always * has "type" and "urls", and often "label", "title" or "description". It - * is not the child node. Use DasNode_subNode() to load that. + * is not the child node. A walk, or a session's DasCat_subNode(), loads that. * * @returns the stub dictionary, or NULL if the index is out of range * @memberof DasNode @@ -478,25 +294,211 @@ DAS_API das_node_type_e DasNode_subType(const DasNode* pThis, size_t uIdx); /** If true the internal data for this node are in JSON format - * - * @param pThis The catalog node - * @return True if DasNode_getJdo() usable + * + * Always true today; the hook is for catalogs in another format. + * @param pThis The catalog node + * @return True if DasNode_getJdo() is usable + * @memberof DasNode */ DAS_API bool DasNode_isJson(const DasNode* pThis); -/** Get a JSON document object at a fragment location in a node. +/** Get a JSON document object at a fragment location in a node. * @param pThis The catalog node - * @param sFragment The fragment path, for example "CONTACTS/TECH/0/EMAIL" - * @return The json object at that location, or NULL if the node does not + * @param sFragment The fragment path, for example "contacts/0/email", or + * NULL for the whole document + * @return The json object at that location, or NULL if the node does not * contain the given element. * @memberof DasNode */ DAS_API const DasJdo* DasNode_getJdo(const DasNode* pThis, const char* sFragment); + +/* ************************************************************************* */ +/* Walking */ + +/** @addtogroup catalog + * @{ + */ + +/** A walk from a node to a descendant, opaque. + * + * The catalog's path resolution with the transport taken out. A walk + * knows which document it needs next and what to make of the one it is + * handed; whoever drives it reads files or makes HTTP requests. + * + * Each step offers the candidate URLs for one node, the mirrors a catalog + * lists for a child, in catalog order. A sequential driver takes index + * 0, and after a DasNodeWalk_fail() or a document that does not parse the + * list shrinks, so index 0 is always the next thing to try. A driver that + * fetches mirrors in parallel feeds whichever answered. A document that + * parses but leads nowhere, a look-alike child or a mirror out of date, + * is dropped silently and the next candidate offered. When a listing is + * exhausted the walk backs out a level and tries the level above's next + * candidate, so a dead mirror doesn't hide a live one. + * + * Nothing joins the tree until the walk succeeds; then the nodes it + * loaded are attached under their parents and a second walk of the same + * path needs no document at all. A walk that fails or is deleted early + * frees what it loaded. + * + * The driver's loop, for a program that fetches for itself: + * + * @code + DasNodeWalk* pWalk = new_DasNodeWalk(pSite, "juno/wav/survey"); + if(pWalk == NULL) handle_error(); + + while(DasNodeWalk_state(pWalk) == das_walk_fetch){ + + // Index 0 is always the next candidate to try, see below + const char* sUrl = DasNodeWalk_url(pWalk, 0); + + char* pDoc = NULL; + size_t uDoc = 0; + if(my_fetch(sUrl, &pDoc, &uDoc)){ + // DASERR_NODE here means the bytes were not a node; the walk has + // already moved on, so there is nothing to do but note it + if(DasNodeWalk_feed(pWalk, 0, pDoc, uDoc, NULL) != DAS_OKAY) + log("%s did not parse", sUrl); + free(pDoc); + } + else{ + DasNodeWalk_fail(pWalk, 0, "no answer"); + } + } + + DasNode* pSurvey = DasNodeWalk_result(pWalk); + if(pSurvey == NULL){ + const char* sRest = NULL; + const char* sNear = DasNodeWalk_nearest(pWalk, &sRest); + log("nothing at %s below %s", sRest, sNear); + } + del_DasNodeWalk(pWalk); // pSurvey lives on under pSite + @endcode + * + * The same loop loads the compiled in seed, new_DasNodeWalk_seed(): the + * candidates are then the seed URLs, see das_seed_urls(), and the result + * is the root of the das catalog, a seed the caller owns and frees with + * del_DasNode(). A lookup by full URI is a walk from it, and what it + * loads stays under it, so a client keeps the seed for its lifetime and + * the tree below it is its memory of the catalog. In a browser build the + * loop is the same again with an await on fetch() in place of my_fetch(). + */ +typedef struct das_node_walk DasNodeWalk; + +/** What a walk is waiting for */ +typedef enum das_walk_state_enum { + das_walk_fetch = 0, /**< a document, see DasNodeWalk_url() */ + das_walk_done, /**< nothing, DasNodeWalk_result() has the node */ + das_walk_failed /**< nothing, no path led to it, see DasNodeWalk_nearest() */ +} das_walk_state_e; + +/** Start a walk. + * + * @param pFrom The container to start from, borrowed; it must outlive + * the walk. + * + * @param sRelPath The path below pFrom, with or without the separator + * pFrom uses in front. Copied. Empty means pFrom itself and the + * walk is done at once. Not expanded: a full URI is spelled out, + * "tag:das2.org,2012:..." + * + * @returns A new walk, to be freed with del_DasNodeWalk(), already + * advanced to its first state. NULL after logging why not: no + * start node, or one that is not a container. + * @memberof DasNodeWalk + */ +DAS_API DasNodeWalk* new_DasNodeWalk(DasNode* pFrom, const char* sRelPath); + +/** Start a walk that loads the compiled in seed. + * + * One step, whose candidates are the URLs from das_seed_urls() in order, + * so a mirror that does not answer is passed over for the next. The + * result is the index of the das catalog, a root: it has no URI, its + * children are the top level catalogs by URI prefix, and it is the + * caller's to keep and to free with del_DasNode(). Everything else in + * the federated catalog is a walk from it. + * + * @returns A new walk, to be freed with del_DasNodeWalk(), waiting for + * a document. + * @memberof DasNodeWalk + */ +DAS_API DasNodeWalk* new_DasNodeWalk_seed(void); + +/** Free a walk. A finished walk's result stays where the walk attached + * it, or in the caller's hands for a seed. Nodes loaded by a walk that + * did not finish go with it. + * @memberof DasNodeWalk */ +DAS_API void del_DasNodeWalk(DasNodeWalk* pThis); + +/** @memberof DasNodeWalk */ +DAS_API das_walk_state_e DasNodeWalk_state(const DasNodeWalk* pThis); + +/** The number of candidate URLs on offer, 0 unless the state is + * das_walk_fetch. @memberof DasNodeWalk */ +DAS_API size_t DasNodeWalk_urlCount(const DasNodeWalk* pThis); + +/** Candidate number uIdx, borrowed and valid until the next feed or fail, + * or NULL past the end. @memberof DasNodeWalk */ +DAS_API const char* DasNodeWalk_url(const DasNodeWalk* pThis, size_t uIdx); + +/** The catalog URI of the node this step would load, for messages and + * for a driver's cache. "" when loading a seed, which has none. + * @memberof DasNodeWalk */ +DAS_API const char* DasNodeWalk_uri(const DasNodeWalk* pThis); + +/** Hand the walk the document a candidate URL produced. + * + * @param uIdx Which candidate, see DasNodeWalk_url(). Candidates before + * it are dropped as not tried. + * @param pBytes The document text, need not be NUL terminated. + * @param uBytes Its length. + * @param sFinalUrl The URL after redirects, for messages. NULL means + * the candidate itself. + * @returns DAS_OKAY when the text parsed as a node, whether or not it led + * anywhere, and the walk has advanced. DASERR_NODE when it did + * not parse, or is not a Catalog when a seed was wanted, after + * logging; that candidate is dropped and the walk has advanced as + * after DasNodeWalk_fail(). Also DASERR_NODE, with nothing + * changed, when the walk was not waiting or uIdx is out of range. + * @memberof DasNodeWalk + */ +DAS_API DasErrCode DasNodeWalk_feed( + DasNodeWalk* pThis, size_t uIdx, const char* pBytes, size_t uBytes, + const char* sFinalUrl +); + +/** Tell the walk a candidate URL produced nothing. + * @param uIdx Which candidate. It and those before it are dropped. + * @param sWhy For the debug log, may be NULL. + * @memberof DasNodeWalk */ +DAS_API void DasNodeWalk_fail(DasNodeWalk* pThis, size_t uIdx, const char* sWhy); + +/** The node a finished walk reached, or NULL. Attached under the start + * node's tree, or for new_DasNodeWalk_seed() the seed itself, the + * caller's to del_DasNode(). + * @memberof DasNodeWalk */ +DAS_API DasNode* DasNodeWalk_result(const DasNodeWalk* pThis); + +/** After a failure, the deepest container the walk searched. + * + * Only its URI: a node the walk loaded was not kept. A session can load + * it again cheaply and say "no child X under Y, which has ...". An empty + * string names the start node, when that is a root; it is the one + * container the caller still holds. + * + * @param psRest If not NULL, receives the part of the path below that + * container, valid for the life of the walk. + * @returns The URI, or NULL when the state is not das_walk_failed or no + * container was ever searched, as when no seed URL answered. + * @memberof DasNodeWalk */ +DAS_API const char* DasNodeWalk_nearest(const DasNodeWalk* pThis, const char** psRest); + +/** @} */ + /** Free a node a constructor returned, and every node loaded below it. * - * A child node, one from DasNode_subNode(), is ignored: it belongs to its + * A child node, one a walk attached, is ignored: it belongs to its * parent and is freed with it. * @memberof DasNode */ diff --git a/das3/node_priv.h b/das3/node_priv.h index 35507e8..ae97443 100644 --- a/das3/node_priv.h +++ b/das3/node_priv.h @@ -42,38 +42,28 @@ * Loading. Every node begins as a JSON document that _DasNode_fromJson() * turns into a struct of the right type. Where the bytes come from is * not node.c's business: catalog.c reads files, the cache and HTTP, a - * browser build hands them to new_DasNode_json(). The parsed DasJdo is kept in pDom - * for the life of the node and the accessors read from it rather than - * copying fields out. DasNode_subNode() matches a relative path against - * the names in the parent's container dictionary, loads the matching - * child's URL and remembers the child in the parent's pSubNodes array, so - * a second lookup of the same path costs nothing. + * browser build hands them to new_DasNode_json(). The parsed DasJdo is + * kept in pDom for the life of the node and the accessors read from it + * rather than copying fields out. A DasNodeWalk resolves a path below a + * container by matching it against the names in the container dictionary + * and asking its driver for the matching child's URL; a child that arrives + * is remembered in the parent's pSubNodes array once the walk succeeds, + * so a second walk of the same path needs no document. * * Ownership. Trees are owned from the top. A node with bOwner set was * handed to a caller by a constructor: it owns everything loaded below it * and only del_DasNode() frees it. Callers never free a child node, it * belongs to its parent's pSubNodes array. An owner need not be the top - * of any catalog: _DasNode_new() walks down from the federated roots to - * the requested node, marks that one, then deletes the nodes above it - * (_DasNode_delTree() skips owners). An application holding - * site:/uiowa/juno/wav thus owns a one node tree with no parent in memory. - * - * Inherited settings. pMgr, sAgent and pCache are borrowed from whoever - * made the root and copied into each child as it loads, so a DasCat's - * credentials, agent string and cache follow a whole walk without being - * passed at every call. None are owned by the node; the caller keeps them - * alive for as long as the tree exists. + * of any catalog: a seed loaded from a file, or a node a program kept + * after dropping the tree it was found in, since _DasNode_delTree() skips + * owners. An application holding site:/uiowa/juno/wav thus owns a one + * node tree with no parent in memory. + * + * Transport. None. A node holds no credentials, agent string or cache + * settings; the session that loads nodes, see catalog.c, keeps those and + * drives a DasNodeWalk with them. */ -/* Keeps track of downloaded nodes and their age. - * - * A manager object such as DasCat owns node caches, DasNode just uses the cache. - */ -typedef struct das_node_cache { - char sDir[256]; /* empty means no caching */ - int nMaxAgeSec; /* zero or less means always ask the server, see _DasNode_mkNode() */ -} DasNodeCache; - struct das_node { das_node_type_e nType; char sURL[512]; /* where this node was loaded from */ @@ -83,9 +73,6 @@ struct das_node { void* pDom; /* the parsed document, a DasJdo, current layout */ void* pShim; /* elements added on load that the parser did not allocate, see _DasNode_upgrade(). May be NULL */ - DasCredMngr* pMgr; /* borrowed from the loader, inherited by children */ - const char* sAgent; /* borrowed, usually a string literal */ - const DasNodeCache* pCache; /* borrowed, inherited by children */ }; /* Catalog nodes have extra arrays to cache nodes */ @@ -131,10 +118,12 @@ void _DasNode_delTree(DasNode* pThis); /* Catalog URI spellings. * - * Names are composed top down: the das2 root node has no separator, so - * its child is "tag:das2.org,2012:site", and that node's own separator - * ":/" makes "tag:das2.org,2012:site:/uiowa". The shorthands "site:", - * "test:" and "local:" follow suit. Alone, one names the node: + * Names are composed top down: the das catalog root node has no separator, so + * its child is "tag:das2.org,2012:site", and that node's separator + * ":/" makes "tag:das2.org,2012:site:/uiowa". A root has no URI, so its + * children are named by their bare names and the levels below by the + * path from the root. The shorthands "site:" and "test:" follow the + * composition rule. Alone, one names the node: * "site:" is "tag:das2.org,2012:site". With a path after it, it stands * for the node and its separator, and a '/' is supplied or collapsed so * that "site:uiowa" and "site:/uiowa" name one node and stamp one @@ -143,130 +132,37 @@ void _DasNode_delTree(DasNode* pThis); * and may not be '/'. Returns sOut, empty for a NULL input. */ const char* _das_uri_expand(const char* sIn, char* sOut, size_t uOut); -/* The local: name of a node loaded by location rather than found in a - * catalog: D2URI_LOCAL followed by the location. A relative file path is - * prefixed with the working directory, nothing more; symbolic links are - * not resolved, the spelling the user gave is the name. A file:// URL - * loses its scheme, an http(s) URL loses its scheme and gets a '/' before - * the host. Returns sOut. */ -const char* _das_uri_local(const char* sLoc, char* sOut, size_t uOut); - -/* True for URIs under D2URI_LOCAL. These resolve only through a session's - * own mounts, never through the federated roots. */ -bool _das_uri_isLocal(const char* sUri); - -/* Constructors. The public new_DasNode() and new_DasNode_url() in - * node.h are thin wrappers that pass NULL for pCache, so trees made through - * them are never cached. A DasCat passes its own settings; the session is - * the only place caching is configured. */ - -/* Load the node at a catalog path URI by walking down from the built in - * federated roots, see das_root_urls(). - * - * Each root URL is tried in turn with a short connect timeout and the whole - * list is retried with longer ones before giving up, so one slow mirror - * does not stall a walk. The node found is marked as the caller's and - * every node above it is freed, see Ownership above. Credentials are not - * offered to the top level roots, only to the nodes below them, which is - * where restricted catalogs live. - * - * @param sPathUri The full catalog URI of the node wanted, for example - * "tag:das2.org,2012:site:/uiowa/juno/wav". The "site:" and "test:" - * shorthands are expanded to full tag URIs first. NULL returns the - * federated root node itself. - * - * @param pMgr The credentials manager offered to nodes below the top level, - * and inherited by every node loaded later from the result. May be - * NULL if no restricted catalogs will be walked. Borrowed. - * - * @param sAgent The HTTP user agent string sent with every request, also - * inherited. Borrowed, usually a string literal. - * - * @param pCache Where fetched node documents are saved and how long they - * are trusted, inherited by the whole tree. NULL means no caching. - * Borrowed. - * - * @returns A node the caller owns, to be freed with del_DasNode(), or NULL - * if no federated root could be reached or none lead to sPathUri. - * Failures are logged, not raised through das_error(). - */ -DasNode* _DasNode_new( - const char* sPathUri, DasCredMngr* pMgr, const char* sAgent, - const DasNodeCache* pCache -); - -/* Load one node directly from a URL or file path for the caller to own. - * - * The maximum connect timeout is used since there is no alternate URL to - * fall back on. - * - * @param sUrl An http://, https:// or file:// URL, or a bare file system - * path, of a node document. Local files are read directly and never - * cached. - * - * @param sPathUri The catalog URI to record for the node, which the loader - * has no way to know from the URL alone. It may be any name the - * caller likes, shorthands expanded, or NULL or "" to name the node - * by its location under D2URI_LOCAL. Children extend it using the - * node's own separator. - * - * @param pMgr Credentials for this fetch and for every node loaded later - * from the result. May be NULL. Borrowed. - * - * @param sAgent The HTTP user agent string, inherited by children. - * Borrowed, usually a string literal. - * - * @param pCache Cache settings for the tree, NULL for no caching. - * Borrowed. - * - * @returns A node the caller owns, to be freed with del_DasNode(), or NULL - * if the document could not be read or is not a recognized node type. - * Failures are logged, not raised through das_error(). - */ -DasNode* _DasNode_newUrl( - const char* sUrl, const char* sPathUri, DasCredMngr* pMgr, const char* sAgent, - const DasNodeCache* pCache -); - -/* A synthetic Catalog node whose children are other nodes placed at chosen - * catalog URIs. This is the machinery behind DasCat_mount() and the named - * form of DasCat_setRoot(). +/* The mount table: a synthetic Catalog node whose children are other + * nodes placed at chosen catalog URIs. This is the machinery behind + * DasCat_mount() and DasCat_setSeed(). * - * It works the way the federated root catalog at das2.org does. That node - * has "separator":null, so its children are named by whole URI prefixes - * such as "tag:das2.org,2012:" and node.c cannot split a path on a - * separator to find one. Instead _DasNode_subNode() walks the children in - * document order and takes the FIRST child whose name is a prefix of the - * requested path, then resolves the rest of the path below that child. + * It works the way the das catalog index does. That node has + * "separator":null, so its children are named by whole URI prefixes such + * as "tag:das2.org,2012:" and a path cannot be split on a separator to + * find one. Instead the walk goes through the children in document order + * and takes the FIRST child whose name is a prefix of the requested path, + * then resolves the rest of the path below that child. * - * A mount root is a small JSON document of that shape, written in memory - * and parsed like any downloaded node. Each child is named by the full URI - * of a mount and its urls list holds the file or URL to load there. For - * the two mounts (./uiowa.json at site:/uiowa) and (./wav.json at - * site:/uiowa/juno/wav) with fall-through, the document is in spirit: + * A mount table is a small JSON document of that shape, written in memory + * and parsed like any downloaded node. Each child is named by the full + * URI of a mount and its urls list holds the file or URL to load there. + * For the two mounts (./uiowa.json at site:/uiowa) and (./wav.json at + * site:/uiowa/juno/wav) the document is in spirit: * * { "type":"Catalog", "separator":null, "catalog":{ * "tag:das2.org,2012:site:/uiowa/juno/wav": {"urls":["./wav.json"]}, - * "tag:das2.org,2012:site:/uiowa": {"urls":["./uiowa.json"]}, - * "": {"urls":["https://das2.org/catalog/index.json", ...]} + * "tag:das2.org,2012:site:/uiowa": {"urls":["./uiowa.json"]} * }} * * Resolving site:/uiowa/juno/wav/survey matches the wav child, loads * wav.json and looks up "/survey" inside it. site:/uiowa/cassini skips - * wav and matches uiowa. A CDAWeb URI matches neither mount, but the - * empty string is a prefix of every path, so the last child wins and the - * federated roots take over. - * - * Ordering matters because the first match wins, not the best match. If - * uiowa were listed before wav it would also claim every wav path and - * wav.json would never be consulted. Hence the children are written - * longest URI first, so a more specific mount shadows a broader one. - * - * With bFallThrough the empty-named child is appended and any path no mount - * claims resolves through the normal catalog. Without it the mounts are - * the whole catalog and everything else fails to resolve. A named root is - * exactly that with a single mount, which is how DasCat_setRoot() builds - * one. + * wav and matches uiowa. Ordering matters because the first match wins, + * not the best match. If uiowa were listed before wav it would also + * claim every wav path and wav.json would never be consulted. Hence the + * children are written longest URI first, so a more specific mount + * shadows a broader one. Whether a URI no mount covers goes on to the + * federated catalog is the session's decision, made before the table is + * consulted at all. * * @param aUrl The URL or file path to load for each mount, in the same order * as aUri. None may be NULL or empty. @@ -279,29 +175,12 @@ DasNode* _DasNode_newUrl( * @param uMounts The number of entries in aUrl and aUri, at least 1 and at * most 64, the size of the sort buffer in node.c. * - * @param bFallThrough If true, paths no mount claims resolve through the - * built in federated roots. If false the mounts are the whole - * catalog. - * - * @param pMgr Credentials offered when the mounted nodes and their children - * are loaded. May be NULL. Borrowed. - * - * @param sAgent The HTTP user agent string, inherited by children. - * Borrowed, usually a string literal. - * - * @param pCache Cache settings for the tree, NULL for no caching. - * Borrowed. - * - * @returns A new mount root, owned by the caller, whose source URL reports + * @returns A new mount table, owned by the caller, whose source URL reports * as "memory:mounts", to be freed with del_DasNode(), or NULL if the * arguments are bad * or the document could not be allocated. Nothing is fetched at * this point; the mounted nodes load on first lookup. */ -DasNode* _DasNode_newMountRoot( - const char** aUrl, const char** aUri, size_t uMounts, - bool bFallThrough, DasCredMngr* pMgr, const char* sAgent, - const DasNodeCache* pCache -); +DasNode* _DasNode_newMountTable(const char** aUrl, const char** aUri, size_t uMounts); #endif /* _das_node_priv_h_ */ diff --git a/das3/source.c b/das3/source.c index f9bf3d7..0428e04 100644 --- a/das3/source.c +++ b/das3/source.c @@ -40,7 +40,6 @@ #include "util.h" #include "log.h" #include "time.h" -#include "http.h" #include "node.h" #include "node_priv.h" #include "src_priv.h" @@ -108,7 +107,7 @@ static char* _strdupz(const char* s) } /* Every string a source keeps, keys, default values, enum items and range - * ends, is its own small allocation, freed with the source. A possible + * ends, is a separate small allocation, freed with the source. A possible * optimization, not taken: the source owns pDoc for its whole life, so * plain JSON strings could be referenced in place and only numbers, which * need formatting, copied. That would remove most of these allocations @@ -223,7 +222,7 @@ static bool _typeByName(const char* s, das_srcprop_type_e* pType) return false; } -/* Preference order: a set enum list, the property's own type, a boolean +/* Preference order: a set enum list, the property's type, a boolean * default, a flag toggle, the protocol parameter's type, the shape of the * value, then text. The enum list goes first because a property declared * "string" with a list of choices is still a list of choices to anyone @@ -302,7 +301,7 @@ static void _clearCtrl(DasSrcCtrl* p) /* A member of a group dict is a property if it is a dict that looks like * one. In the current layout every member of "props" qualifies; in the - * 2018 layout properties sit beside the group's own name and title + * 2018 layout properties sit beside the group's name and title * strings, and a units dict with a value counts as the units property * just as it does under props today. */ static bool _isPropDict(const DasJdo* p) @@ -706,15 +705,11 @@ DasSrc* new_DasSrc_json(const char* sDoc, size_t uLen, const char* sPathUri, con pThis->kind = das_src_kind_stream; pThis->pDoc = pDoc; - /* Every source has a URI, see D2URI_LOCAL. One from a bare document - * with no location is "local:/memory", which is all that can be said */ - char sUri[600]; + /* A source given no URI has none, and a requisition it makes carries + * no stamp, see DasReq_setSrc(): nothing can hold it to one source */ + char sUri[600] = {'\0'}; if((sPathUri != NULL) && (sPathUri[0] != '\0')) _das_uri_expand(sPathUri, sUri, sizeof(sUri)); - else if((sUrl != NULL) && (sUrl[0] != '\0')) - _das_uri_local(sUrl, sUri, sizeof(sUri)); - else - strcpy(sUri, D2URI_LOCAL "/memory"); pThis->sPath = _strdupz(sUri); pThis->sUrl = _strdupz(sUrl); if((pThis->sPath == NULL)||(pThis->sUrl == NULL)){ del_DasSrc(pThis); return NULL; } @@ -1503,7 +1498,7 @@ static void _outClear(_out_t* pO) /* One control's value to protocol parameters, by its set block: * * - enum: the chosen item's "pval" if it has one, else its value, and - * an item may name its own flag + * an item may name a flag * - boolean: the block fires for one state only, "value" (true unless * said otherwise); the other state is the default and sends nothing. * With neither pval nor flag the state itself is sent as true/false. diff --git a/das3/source.h b/das3/source.h index 82cc722..c2e42ee 100644 --- a/das3/source.h +++ b/das3/source.h @@ -256,8 +256,7 @@ DAS_API DasSrc* new_DasSrc_node(const DasNode* pNode); * @param sPathUri The catalog URI of the node, shorthands accepted. It * is the source's identity: requisitions are stamped with it and * later checked against it, see DasSrc_check(). NULL or empty - * names the source after sUrl under D2URI_LOCAL, or as - * "local:/memory" when there is no URL either. + * leaves the source without one, and its requisitions unstamped. * @param sUrl Where the text came from, recorded as the source's URL and * used to resolve relative links. May be NULL. * @returns A new source, or NULL after logging a parse problem or a @@ -272,8 +271,7 @@ DAS_API void del_DasSrc(DasSrc* pThis); /** @memberof DasSrc */ DAS_API das_src_kind_e DasSrc_kind(const DasSrc* pThis); -/** The catalog URI, the source's identity. Never empty: a source loaded - * by location with no URI given is named under D2URI_LOCAL. +/** The catalog URI, the source's identity, or "" for a source given none * @memberof DasSrc */ DAS_API const char* DasSrc_path(const DasSrc* pThis); diff --git a/das3/util.c b/das3/util.c index 3536aef..6a7de3c 100644 --- a/das3/util.c +++ b/das3/util.c @@ -1220,3 +1220,55 @@ const char* das_stream_type_toStr(das_stream_type type) default: return "unknown"; } } + +/* ************************************************************************* */ +/* Strings that name places: URLs, files and catalog URIs */ + +size_t das_url_encode(const char* sIn, char* sOut, size_t uOut) +{ + static const char* sHex = "0123456789ABCDEF"; + size_t uNeed = 0; + if(uOut > 0) sOut[0] = '\0'; + for(const unsigned char* p = (const unsigned char*)sIn; p && *p; ++p){ + bool bPlain = isalnum(*p)||(*p == '-')||(*p == '.')||(*p == '_')||(*p == '~'); + size_t n = bPlain ? 1 : 3; + if(uNeed + n < uOut){ + if(bPlain){ + sOut[uNeed] = (char)*p; + } + else{ + sOut[uNeed] = '%'; + sOut[uNeed + 1] = sHex[*p >> 4]; + sOut[uNeed + 2] = sHex[*p & 0x0F]; + } + sOut[uNeed + n] = '\0'; + } + uNeed += n; + } + return uNeed; +} + +static bool _das_hasPfx(const char* s, const char* sPrefix) +{ + return (s != NULL) && (strncmp(s, sPrefix, strlen(sPrefix)) == 0); +} + +bool das_isUri(const char* sPath) +{ + if((sPath == NULL) || !isalpha((unsigned char)sPath[0])) return false; + const char* p = sPath + 1; + while(isalnum((unsigned char)*p) || (*p == '+') || (*p == '-') || (*p == '.')) ++p; + /* A single letter before the colon is a Windows drive, not a scheme */ + return (*p == ':') && (p > sPath + 1); +} + +bool das_isUrlOrFile(const char* sPath) +{ + if(sPath == NULL) return false; + if(_das_hasPfx(sPath, "http://")||_das_hasPfx(sPath, "https://")||_das_hasPfx(sPath, "file://")) return true; + if((sPath[0] == '.')||(sPath[0] == '/')||(sPath[0] == '\\')) return true; + if(isalpha((unsigned char)sPath[0]) && (sPath[1] == ':') && ((sPath[2] == '\\')||(sPath[2] == '/'))) + return true; /* a Windows drive, catalog URIs never have their first colon at index 1 */ + size_t u = strlen(sPath); + return (u > 5) && (strcmp(sPath + u - 5, ".json") == 0); +} diff --git a/das3/util.h b/das3/util.h index aa4e1f7..d913fd7 100644 --- a/das3/util.h +++ b/das3/util.h @@ -442,6 +442,44 @@ DAS_API const char* das_xml_escape(char* dest, const char* src, size_t uOutLen); */ DAS_API char* das_strdup(const char* sIn); +/** Percent encode a string for use in a URL query. + * + * Everything but the RFC 3986 unreserved characters, letters, digits and + * "-._~", is encoded, including the ':' in time strings. Servers decode + * before use so over-encoding is harmless, and it is what most encoders + * do. Works like snprintf: the output is always NUL terminated when uOut + * is at least 1, and the return value says how much room was needed. + * + * @param sIn The text to encode. + * @param sOut Receives the encoding. Three times the input length plus + * one always suffices. + * @param uOut The size of sOut. + * @returns The length of the full encoding, not counting the terminator. + * If this is uOut or more the output was cut short. + */ +DAS_API size_t das_url_encode(const char* sIn, char* sOut, size_t uOut); + +/** Is this string a node location rather than a catalog path? + * + * True for http://, https:// and file:// URLs, for anything starting with + * '.', '/', '\' or a drive letter, and for anything ending in ".json". All else + * is taken to be a catalog path or URI. This is the rule DasCat_resolve() + * uses, published so command line parsers can agree with it. + * + */ +DAS_API bool das_isUrlOrFile(const char* sPath); + +/** Is this string a full URI rather than a relative catalog path? + * + * True when it starts with a URI scheme, a letter followed by letters, + * digits, '+', '-' or '.' and then a colon, as in "tag:", "doi:" and the + * shorthands "site:" and "test:". A single letter before the colon is a + * Windows drive, not a scheme. Note that URLs are URIs too; test with + * das_isUrlOrFile() first when the difference matters. + * + */ +DAS_API bool das_isUri(const char* sPath); + /** A memset that handles multi-byte items * * Uses memcpy because the amount of data written in each call goes up diff --git a/test/TestCat.c b/test/TestCat.c index 7cca3d5..90d82cd 100644 --- a/test/TestCat.c +++ b/test/TestCat.c @@ -50,16 +50,6 @@ fixtures are local files, which are never cached. */ #define TEST_USERDIR "test_das_userdir_never_created" -/* The local: name a relative test file gets: the working directory is - * part of it, see D2URI_LOCAL. Static buffer, one call per CHECK. */ -static const char* localUri(const char* sFile) -{ - static char sBuf[1024]; - char sCwd[512] = {'\0'}; - if(getcwd(sCwd, sizeof(sCwd) - 1) == NULL) sCwd[0] = '\0'; - snprintf(sBuf, sizeof(sBuf), D2URI_LOCAL "%s/%s", sCwd, sFile); - return sBuf; -} static int g_nFail = 0; static int g_nPass = 0; @@ -121,7 +111,7 @@ static void test_isUrlOrFile(void) printf("TEST: das_isUri\n"); CHECK(das_isUri("tag:das2.org,2012:site:/uiowa"), "tag URI"); CHECK(das_isUri("doi:10.1037/a0040251/"), "doi URI"); - CHECK(das_isUri("site:/uiowa") && das_isUri("test:x") && das_isUri("local:/x"), "shorthands"); + CHECK(das_isUri("site:/uiowa") && das_isUri("test:x"), "shorthands"); CHECK(das_isUri("https://host/x.json"), "a URL is a URI too"); CHECK(!das_isUri("uiowa/juno/wav:x"), "relative path, colon after a slash"); CHECK(!das_isUri("C:\\nodes\\x") && !das_isUri("C:/x"), "drive letters are not schemes"); @@ -131,15 +121,15 @@ static void test_isUrlOrFile(void) /* ************************************************************************* */ static void test_policy(void) { - printf("TEST: root policy setters\n"); + printf("TEST: prefix, seed and mount setters\n"); DasCat* pCat = new_DasCat(NULL, "TestCat"); CHECK(pCat != NULL, "new_DasCat"); CHECK(DasCat_credMngr(pCat) == NULL, "no manager"); CHECK(streq(DasCat_agent(pCat), "TestCat"), "agent copied"); CHECK(streq(DasCat_prefix(pCat), DASCAT_DEF_PREFIX), "default prefix"); - CHECK(!DasCat_hasRoot(pCat), "no root yet"); - CHECK(DasCat_rootUrl(pCat) == NULL, "no root url"); - CHECK(DasCat_rootUri(pCat) == NULL, "no root URI"); + CHECK(!DasCat_hasSeed(pCat), "no seed yet"); + CHECK(DasCat_seedUrl(pCat) == NULL, "no seed url"); + CHECK(streq(DasCat_prefix(pCat), DASCAT_DEF_PREFIX), "no seed: paths start at the default prefix"); CHECK(DasCat_mountCount(pCat) == 0, "no mounts"); CHECK(DasCat_lastUri(pCat) == NULL, "no resolve yet"); @@ -153,16 +143,17 @@ static void test_policy(void) CHECK(quiet(DasCat_setPrefix, pCat, SITE) != DAS_OKAY, "a file is not a prefix"); CHECK(quiet(DasCat_setPrefix, pCat, "") != DAS_OKAY, "empty prefix"); - CHECK(quiet2(DasCat_setRoot, pCat, "uiowa", NULL) != DAS_OKAY, "a path is not a root"); - CHECK(DasCat_setRoot(pCat, SITE, NULL) == DAS_OKAY, "root by file"); - CHECK(DasCat_hasRoot(pCat), "has root"); - CHECK(streq(DasCat_rootUrl(pCat), SITE), "root url kept"); - CHECK(quiet2(DasCat_setRoot, pCat, UIOWA, NULL) != DAS_OKAY, "one root only"); - CHECK(quiet2(DasCat_setRoot, pCat, UIOWA, UIOWA_URI) != DAS_OKAY, "no named root over a root"); - CHECK(streq(DasCat_rootUri(pCat), localUri(SITE)), "a root without a URI is named by location: %s", - DasCat_rootUri(pCat)); + CHECK(quiet2(DasCat_setSeed, pCat, "uiowa", NULL) != DAS_OKAY, "a path is not a seed"); + CHECK(DasCat_setSeed(pCat, SITE, NULL) == DAS_OKAY, "seed by file"); + CHECK(DasCat_hasSeed(pCat), "has seed"); + CHECK(streq(DasCat_seedUrl(pCat), SITE), "seed url kept"); + CHECK(quiet2(DasCat_setSeed, pCat, UIOWA, NULL) != DAS_OKAY, "one seed only"); + CHECK(quiet2(DasCat_setSeed, pCat, UIOWA, UIOWA_URI) != DAS_OKAY, "no second seed with a prefix either"); + CHECK(quiet(DasCat_setPrefix, pCat, "site:/") != DAS_OKAY, "no prefix once a seed is set, paths start at the seed"); + CHECK(streq(DasCat_prefix(pCat), ""), "the seed superseded the prefix set before it"); + CHECK(streq(DasCat_prefix(pCat), ""), "a seed without a prefix is a root, paths start at nothing"); CHECK(DasCat_mount(pCat, UIOWA, UIOWA_URI) == DAS_OKAY, "a mount joins an offline catalog"); - CHECK(DasCat_mountCount(pCat) == 2, "the root is a mount too"); + CHECK(DasCat_mountCount(pCat) == 1, "the seed is not a mount"); CHECK(DasCat_covers(pCat, UIOWA_URI "/x") && !DasCat_covers(pCat, "tag:das2.org,2012:site:/swri"), "covered by the mount, walled off elsewhere"); del_DasCat(pCat); @@ -170,43 +161,41 @@ static void test_policy(void) pCat = new_DasCat(NULL, NULL); CHECK(DasCat_agent(pCat) == NULL, "NULL agent stays NULL"); CHECK(quiet2(DasCat_mount, pCat, UIOWA, "") != DAS_OKAY, "a mount needs a URI"); - CHECK(quiet2(DasCat_setRoot, pCat, UIOWA, UIOWA) != DAS_OKAY, "root URI can't be a file"); - CHECK(quiet2(DasCat_setRoot, pCat, "uiowa", UIOWA_URI) != DAS_OKAY, "root URL must be a location"); - CHECK(DasCat_setRoot(pCat, UIOWA, "site:/uiowa") == DAS_OKAY, "named root"); - CHECK(streq(DasCat_rootUri(pCat), UIOWA_URI), "root URI expanded"); - CHECK(DasCat_hasRoot(pCat), "a named root is a root"); - CHECK(quiet2(DasCat_setRoot, pCat, SITE, NULL) != DAS_OKAY, "no second root"); + CHECK(quiet2(DasCat_setSeed, pCat, UIOWA, UIOWA) != DAS_OKAY, "seed prefix can't be a file"); + CHECK(quiet2(DasCat_setSeed, pCat, "uiowa", UIOWA_URI) != DAS_OKAY, "seed URL must be a location"); + CHECK(DasCat_setSeed(pCat, UIOWA, "site:/uiowa") == DAS_OKAY, "seed with a prefix"); + CHECK(streq(DasCat_prefix(pCat), UIOWA_URI), "seed prefix expanded, and it is where paths start"); + CHECK(DasCat_hasSeed(pCat), "a seed with a prefix is a seed"); + CHECK(quiet2(DasCat_setSeed, pCat, SITE, NULL) != DAS_OKAY, "no second seed"); del_DasCat(pCat); pCat = new_DasCat(NULL, NULL); CHECK(DasCat_mount(pCat, UIOWA, UIOWA_URI) == DAS_OKAY, "mount one"); CHECK(DasCat_mount(pCat, SITE, "site:swri") == DAS_OKAY, "mount two, slash left out"); CHECK(DasCat_mountCount(pCat) == 2, "two mounts"); - CHECK(!DasCat_hasRoot(pCat), "mounts are not a root"); + CHECK(!DasCat_hasSeed(pCat), "mounts are not a seed"); CHECK(DasCat_covers(pCat, "tag:das2.org,2012:site:/swri/x"), "slash supplied in the mount URI"); CHECK(DasCat_covers(pCat, "tag:das2.org,2012:site:/lanl"), "mounts fall through"); - CHECK(!DasCat_covers(pCat, "tag:das2.org,2012:local:/nowhere"), "local: never falls through"); - CHECK(DasCat_setRoot(pCat, SITE, "test:/") == DAS_OKAY, "a root after mounts isolates them"); - CHECK(DasCat_hasRoot(pCat) && (DasCat_mountCount(pCat) == 3), "three mounts, one is the root"); + CHECK(DasCat_setSeed(pCat, SITE, "test:/") == DAS_OKAY, "a seed after mounts isolates them"); + CHECK(DasCat_hasSeed(pCat) && (DasCat_mountCount(pCat) == 2), "two mounts, the seed is not one"); CHECK(!DasCat_covers(pCat, "tag:das2.org,2012:site:/lanl"), "no fall through now"); del_DasCat(pCat); } /* ************************************************************************* */ -static void test_rootMode(void) +static void test_seed(void) { - printf("TEST: a node file as the only root\n"); + printf("TEST: a node file as the seed\n"); DasCat* pCat = new_DasCat(NULL, "TestCat"); DasCat_setCache(pCat, NULL, 0); /* explicitly off, file nodes never cache anyway */ - CHECK(DasCat_setRoot(pCat, SITE, NULL) == DAS_OKAY, "set root"); - char sRootUri[1024]; - strcpy(sRootUri, localUri(SITE)); + CHECK(DasCat_setSeed(pCat, SITE, NULL) == DAS_OKAY, "set seed"); DasNode* pRoot = DasCat_resolve(pCat, NULL); CHECK(pRoot != NULL, "root resolves"); CHECK(pRoot && DasNode_isCatalog(pRoot), "root is a catalog"); - CHECK(streq(DasCat_lastUri(pCat), sRootUri), "no path: last URI is the root's local: name"); - CHECK(pRoot && streq(DasNode_pathUri(pRoot), sRootUri), "the node carries it"); + CHECK(streq(DasCat_lastUri(pCat), ""), "no path: last URI is empty, a root has none"); + CHECK(pRoot && (DasNode_pathUri(pRoot) == NULL), "the node has no URI"); + CHECK(DasCat_seed(pCat) == pRoot, "the seed accessor is the same node"); CHECK(DasCat_resolve(pCat, "") == pRoot, "empty path is the same as NULL"); /* Policy is frozen now */ @@ -216,10 +205,10 @@ static void test_rootMode(void) CHECK(pSrc != NULL, "deep path resolves"); CHECK(pSrc && DasNode_isStreamSrc(pSrc), "leaf is a stream source"); char sExpect[1100]; - /* site.json declares the ":/" separator of the node it stands in for, - and that is what joins its children to whatever name the root got */ - snprintf(sExpect, sizeof(sExpect), "%s:/uiowa/geomag/flex", sRootUri); - CHECK(streq(DasCat_lastUri(pCat), sExpect), "relative path hangs off the local: name: %s", + /* Below a root, names are the path from it: the root's ":/" + separator is not used since there is nothing to join to */ + strcpy(sExpect, "uiowa/geomag/flex"); + CHECK(streq(DasCat_lastUri(pCat), sExpect), "relative path is the name below a root: %s", DasCat_lastUri(pCat)); CHECK(pSrc && streq(DasNode_pathUri(pSrc), sExpect), "node path URI: %s", pSrc ? DasNode_pathUri(pSrc) : "(null)"); @@ -232,15 +221,15 @@ static void test_rootMode(void) /* Misses, and what the catalog can say about them */ CHECK(quietResolve(pCat, "uiowa/nope/deeper") == NULL, "bad child fails"); - snprintf(sExpect, sizeof(sExpect), "%s:/uiowa/nope/deeper", sRootUri); + strcpy(sExpect, "uiowa/nope/deeper"); CHECK(streq(DasCat_lastUri(pCat), sExpect), "last URI is the failed path"); CHECK(DasCat_covers(pCat, DasCat_lastUri(pCat)), "under the root is covered"); - CHECK(!DasCat_covers(pCat, UIOWA_URI), "the federated catalog is not"); + CHECK(!DasCat_covers(pCat, UIOWA_URI), "a URI with a scheme is not under a root"); const char* sRest = NULL; DasNode* pNear = DasCat_nearest(pCat, &sRest); CHECK(pNear == pUiowa, "nearest is the deepest catalog that loaded"); CHECK(streq(sRest, "nope/deeper"), "rest below it: %s", sRest ? sRest : "(null)"); - snprintf(sExpect, sizeof(sExpect), "%s:/uiowa", sRootUri); + strcpy(sExpect, "uiowa"); CHECK(streq(DasCat_lastUri(pCat), sExpect), "last URI now names the nearest node"); CHECK(quietResolve(pCat, "uiowa/broken") == NULL, "listed child with a missing file fails"); @@ -251,43 +240,30 @@ static void test_rootMode(void) CHECK(quietResolve(pCat, "nope") == NULL, "bad child of the root"); pNear = DasCat_nearest(pCat, &sRest); - CHECK(pNear == NULL, "nothing above a root child"); - snprintf(sExpect, sizeof(sExpect), "%s:/nope", sRootUri); - CHECK(streq(DasCat_lastUri(pCat), sExpect), "a failed nearest keeps the failed URI"); + CHECK((pNear == pRoot) && streq(sRest, "nope"), "nearest for a root child is the root itself"); + CHECK(streq(DasCat_lastUri(pCat), ""), "last URI is empty after a root is nearest"); /* A good resolve leaves nothing for nearest to find */ CHECK(DasCat_resolve(pCat, "uiowa/geomag") != NULL, "source set resolves"); - CHECK(DasCat_nearest(pCat, &sRest) == pUiowa, "nearest after success is the parent, by design"); + CHECK(DasCat_nearest(pCat, &sRest) == NULL, "nearest after success is nothing"); - /* A location given as the path gets a local: name */ + /* A location given as the path is a root */ DasNode* pDirect = DasCat_resolve(pCat, UIOWA); CHECK(pDirect && DasNode_isCatalog(pDirect), "node by file path"); - CHECK(streq(DasCat_lastUri(pCat), localUri(UIOWA)), "named by location: %s", DasCat_lastUri(pCat)); - CHECK(pDirect && streq(DasNode_pathUri(pDirect), localUri(UIOWA)), "node path URI is the local: name"); + CHECK(streq(DasCat_lastUri(pCat), UIOWA), "the location stands in for a name: %s", DasCat_lastUri(pCat)); + CHECK(pDirect && (DasNode_pathUri(pDirect) == NULL), "a node loaded by location has no URI"); CHECK(DasCat_nearest(pCat, &sRest) == NULL, "nearest means nothing for a location"); - CHECK(DasCat_resolve(pCat, "./" UIOWA) != pDirect, "a different spelling is a different name"); - CHECK(streq(DasCat_lastUri(pCat), localUri(UIOWA)), "but a leading ./ is dropped: %s", DasCat_lastUri(pCat)); - - /* local: names that nothing is mounted at fail without a lookup */ - CHECK(quietResolve(pCat, "local:/nowhere/x") == NULL, "unmounted local: name fails"); - CHECK(streq(DasCat_lastUri(pCat), "tag:das2.org,2012:local:/nowhere/x"), "and was expanded first"); - - del_DasCat(pCat); + CHECK(DasCat_resolve(pCat, "./" UIOWA) != pDirect, "a different spelling is a different load"); - /* No root at all: a local: name still never reaches the network */ - pCat = new_DasCat(NULL, "TestCat"); - CHECK(quietResolve(pCat, "tag:das2.org,2012:local:/home/nobody/x.json") == NULL, "unrouted"); - CHECK(!DasCat_covers(pCat, "local:/x") || true, "covers takes shorthand too"); - CHECK(!DasCat_covers(pCat, "tag:das2.org,2012:local:/x"), "and says it is not reachable"); del_DasCat(pCat); } /* ************************************************************************* */ -static void test_namedRoot(void) +static void test_prefixedSeed(void) { - printf("TEST: a named root\n"); + printf("TEST: a seed with a prefix\n"); DasCat* pCat = new_DasCat(NULL, "TestCat"); - CHECK(DasCat_setRoot(pCat, UIOWA, "site:/uiowa") == DAS_OKAY, "named root"); + CHECK(DasCat_setSeed(pCat, UIOWA, "site:/uiowa") == DAS_OKAY, "seed with a prefix"); DasNode* pTop = DasCat_resolve(pCat, NULL); CHECK(pTop && DasNode_isCatalog(pTop), "no path: the named node"); @@ -317,11 +293,11 @@ static void test_namedRoot(void) del_DasCat(pCat); /* The site file as the site node: the bare shorthand names the node, - the node's own separator builds the children, so names come out as + the node's separator builds the children, so names come out as the live catalog spells them */ pCat = new_DasCat(NULL, "TestCat"); - CHECK(DasCat_setRoot(pCat, SITE, "site:") == DAS_OKAY, "root named by bare shorthand"); - CHECK(streq(DasCat_rootUri(pCat), D2URI_SITES), "bare shorthand is the node name: %s", DasCat_rootUri(pCat)); + CHECK(DasCat_setSeed(pCat, SITE, "site:") == DAS_OKAY, "seed named by bare shorthand"); + CHECK(streq(DasCat_prefix(pCat), D2URI_SITES), "bare shorthand is the node name: %s", DasCat_prefix(pCat)); pSrc = DasCat_resolve(pCat, "uiowa/geomag/flex"); CHECK(pSrc && streq(DasNode_pathUri(pSrc), UIOWA_URI "/geomag/flex"), "children spelled as the live catalog: %s", pSrc ? DasNode_pathUri(pSrc) : "(null)"); @@ -331,6 +307,40 @@ static void test_namedRoot(void) del_DasCat(pCat); } +/* ************************************************************************* */ +static void test_mountUnderSeed(void) +{ + printf("TEST: mounts in a seed's name space\n"); + const char* sGeomag = "test/das_fetch/geomag_flex_0.7.json"; + + /* A root seed: names are paths from it, and a mount at one of them is + consulted before the listing, however the path is spelled */ + DasCat* pCat = new_DasCat(NULL, "TestCat"); + CHECK(DasCat_setSeed(pCat, SITE, NULL) == DAS_OKAY, "root seed"); + CHECK(DasCat_mount(pCat, sGeomag, "uiowa/broken") == DAS_OKAY, "mount at a path from the root"); + DasNode* pNode = DasCat_resolve(pCat, "uiowa/broken"); + CHECK(pNode && DasNode_isStreamSrc(pNode), "the mount shadows a listed child whose file is missing"); + CHECK(streq(DasCat_lastUri(pCat), "uiowa/broken"), "named by the path: %s", DasCat_lastUri(pCat)); + CHECK(pNode && streq(DasNode_pathUri(pNode), "uiowa/broken"), "the node carries the mount name"); + CHECK(DasCat_resolve(pCat, "uiowa/broken") == pNode, "and is kept"); + DasNode* pFlex = DasCat_resolve(pCat, "uiowa/geomag/flex"); + CHECK(pFlex && DasNode_isStreamSrc(pFlex) && (pFlex != pNode), "paths off the mount still walk the seed"); + CHECK(DasCat_covers(pCat, "uiowa/broken/deeper"), "under the mount is covered"); + del_DasCat(pCat); + + /* A prefixed seed: the relative spelling and the full URI reach the + same mount */ + pCat = new_DasCat(NULL, "TestCat"); + CHECK(DasCat_setSeed(pCat, UIOWA, "site:/uiowa") == DAS_OKAY, "prefixed seed"); + CHECK(DasCat_mount(pCat, sGeomag, "site:/uiowa/broken") == DAS_OKAY, "mount under the prefix"); + pNode = DasCat_resolve(pCat, "broken"); + CHECK(pNode && DasNode_isStreamSrc(pNode), "relative path through the mount"); + CHECK(streq(DasCat_lastUri(pCat), UIOWA_URI "/broken"), "composed name: %s", DasCat_lastUri(pCat)); + CHECK(DasCat_resolve(pCat, UIOWA_URI "/broken") == pNode, "full URI is the same node"); + CHECK(DasCat_resolve(pCat, "site:/uiowa/broken") == pNode, "shorthand too"); + del_DasCat(pCat); +} + /* ************************************************************************* */ static void test_mounts(void) { @@ -338,9 +348,11 @@ static void test_mounts(void) DasCat* pCat = new_DasCat(NULL, "TestCat"); CHECK(DasCat_mount(pCat, UIOWA, "site:/uiowa") == DAS_OKAY, "mount"); - /* Without a prefix relative paths start at the first mount */ + /* A mount does not move where relative paths start; the prefix does, + and a path from there lands on the mount */ + CHECK(DasCat_setPrefix(pCat, "site:/uiowa/") == DAS_OKAY, "prefix at the mount, with its separator"); DasNode* pSrc = DasCat_resolve(pCat, "geomag/flex"); - CHECK(pSrc && DasNode_isStreamSrc(pSrc), "relative to the mount"); + CHECK(pSrc && DasNode_isStreamSrc(pSrc), "relative to the prefix, through the mount"); CHECK(streq(DasCat_lastUri(pCat), UIOWA_URI "/geomag/flex"), "URI under the mount"); /* Full URIs under the mount never touch the network */ @@ -380,7 +392,7 @@ static void test_userdir(void) { printf("TEST: user directory and the session cache default\n"); - /* The default, unless this test's own environment set DAS_USERDIR */ + /* The default, unless this test's environment set DAS_USERDIR */ const char* sEnv = getenv("DAS_USERDIR"); if((sEnv == NULL)||(sEnv[0] == '\0')) CHECK(endsWith(das_userdir(), DAS_DSEPS ".das"), "default is .das under home, got %s", das_userdir()); @@ -430,8 +442,9 @@ int main(int argc, char** argv) test_userdir(); test_isUrlOrFile(); test_policy(); - test_rootMode(); - test_namedRoot(); + test_seed(); + test_prefixedSeed(); + test_mountUnderSeed(); test_mounts(); printf("%d checks passed, %d failed\n", g_nPass, g_nFail); diff --git a/test/TestCatalog.c b/test/TestCatalog.c index ca6dcfb..d71e265 100644 --- a/test/TestCatalog.c +++ b/test/TestCatalog.c @@ -34,6 +34,7 @@ #include #include #include +#include /* An example fully qualified das federated catalog note fragment: @@ -70,30 +71,30 @@ int main(int argc, char** argv) { /* Exit on errors, log info messages and above */ das_init(argv[0], DASERR_DIS_EXIT, 0, DASLOG_INFO, NULL); + + /* Children are loaded by a session, which holds the agent string and + the node cache; the nodes themselves no longer carry either */ + DasCat* pCat = new_DasCat(NULL, sAgent); /* Simple catalog loads, direct URLs */ sUrl = "https://das2.org/catalog/das/site/uiowa/cassini/ephemeris/saturn.json"; - /* In the global catalog this node is name: + /* In the global catalog this node is named: * * tag:das2.org,2012:site:/uiowa/ephemeris/saturn - * - * But were are loading it is if it was an isolated item so we can name it - * whatever we want, hence doggy + * + * But we are loading it by location, as an isolated item, so it is a + * root of its own with no URI. The session owns + * every node it loads; del_DasCat() at the end frees them all. */ - DasNode* pCasEphem = new_DasNode_url(sUrl, "doggy", NULL, sAgent); + DasNode* pCasEphem = DasCat_resolve(pCat, sUrl); nTest += info_or_exit(sUrl, pCasEphem, "Cassini Saturn Ephemeris", nTest); - printf("TEST %d: Deleting a single node", nTest); - del_DasNode(pCasEphem); pCasEphem = NULL; - printf(" [OKAY]\n"); - nTest += 1; - sUrl = "https://das2.org/catalog/das/site/uiowa/juno/wav.json"; - DasNode* pWavCat = new_DasNode_url(sUrl, "whattie", NULL, sAgent); + DasNode* pWavCat = DasCat_resolve(pCat, sUrl); nTest += info_or_exit(sUrl, pWavCat, "Juno Waves Catalog", nTest); - const DasNode* pSurvey = DasNode_subNode(pWavCat, "survey", NULL, sAgent); + const DasNode* pSurvey = DasCat_subNode(pCat, pWavCat, "survey"); nTest += info_or_exit(sUrl, pSurvey, "Juno Waves Survey Data Source", nTest); printf("TEST %d: (Not) Deleting a non-root ", nTest); @@ -103,15 +104,9 @@ int main(int argc, char** argv) { else printf("... [FAILED]"); nTest += 1; - printf("TEST %d: Deleting a catalog node", nTest); - del_DasNode(pWavCat); pWavCat = NULL; pSurvey = NULL; - printf(" [OKAY]\n"); - nTest += 1; - - - /* Central root catalog*/ - DasNode* pRoot = new_DasNode(sUri, NULL, sAgent); - /* Can't call info or exit because root URI is null */ + /* The compiled in seed, the root of the das catalog */ + DasNode* pRoot = DasCat_seed(pCat); + /* Can't call info or exit because the root has no URI */ if(pRoot == NULL){ printf("TEST %d: Find and load root node [FAILED]\n", nTest); return nTest; @@ -123,41 +118,33 @@ int main(int argc, char** argv) { /* Extra cassini check */ sUrl = "tag:das2.org,2012:site:/uiowa/cassini/rpws/survey"; - const DasNode* pCasSur = DasNode_subNode(pRoot, sUrl, NULL, sAgent); + const DasNode* pCasSur = DasCat_subNode(pCat, pRoot, sUrl); nTest += info_or_exit(sUri, pCasSur, "Cassini/survey", nTest); - /*das2 root */ + /*das root */ sUri = "tag:das2.org,2012:"; - const DasNode* pDas2 = DasNode_subNode(pRoot, sUri, NULL, sAgent); + const DasNode* pDas2 = DasCat_subNode(pCat, pRoot, sUri); nTest += info_or_exit(sUri, pDas2, "Root", nTest); /* spase root */ sUri = "tag:spase-group.org,2018:spase://"; - const DasNode* pSpase = DasNode_subNode(pRoot, sUri, NULL, sAgent); + const DasNode* pSpase = DasCat_subNode(pCat, pRoot, sUri); nTest += info_or_exit(sUri, pSpase, "Root", nTest); /* cdaweb root */ sUri = "tag:cdaweb.sci.gsfc.nasa.gov,2018:"; - const DasNode* pCdaWeb = DasNode_subNode(pRoot, sUri, NULL, sAgent); + const DasNode* pCdaWeb = DasCat_subNode(pCat, pRoot, sUri); nTest += info_or_exit(sUri, pCdaWeb, "Root", nTest); /* Deep dig from existing root */ sUri = "tag:das2.org,2012:site:/uiowa/juno/wav/uncalibrated/hrs"; - const DasNode* pWavUcalHrs = DasNode_subNode(pRoot, sUri, NULL, sAgent); + const DasNode* pWavUcalHrs = DasCat_subNode(pCat, pRoot, sUri); nTest += info_or_exit(sUri, pWavUcalHrs, "Juno Waves Uncalibrated HFWBR", nTest); - /* deleting top root */ - printf("TEST %d: Deleting URI based top root ", nTest); - del_DasNode(pRoot); - pRoot = NULL; pDas2 = pSpase = pCdaWeb = pWavUcalHrs = NULL; - printf(" [OKAY]\n"); - nTest += 1; - - - /* direct deep dig to ferret out memory leaks */ + /* A deep path through the session: the levels walked above stay loaded */ sUri = "tag:das2.org,2012:site:/uiowa/juno/wav/survey/das2"; - DasNode* pSurvey2 = new_DasNode(sUri, NULL, sAgent); + DasNode* pSurvey2 = DasCat_resolve(pCat, sUri); nTest += info_or_exit(sUri, pSurvey2, "Waves Survey Deep Read", nTest); /* Get a Query definition */ @@ -172,12 +159,8 @@ int main(int argc, char** argv) { } nTest += 1; - /* And remove it */ - printf("TEST %d: Deleting deep URI direct lookup", nTest); - del_DasNode(pSurvey2); - printf(" [OKAY]\n"); - /* TODO: Add more JSON catalog tests */ + del_DasCat(pCat); return 0; } diff --git a/test/TestSource.c b/test/TestSource.c index f349fcc..f66375a 100644 --- a/test/TestSource.c +++ b/test/TestSource.c @@ -34,6 +34,7 @@ #include #include #include +#include #include "../das3/src_priv.h" #define GEOMAG "test/das_fetch/geomag_flex_0.7.json" @@ -534,9 +535,7 @@ static void test_hsk(void) const char* sName = DasSrc_contact(pSrc, 0, &sEmail); CHECK(sName && sName[0] && sEmail, "contact name %s", sName ? sName : "(none)"); CHECK(DasSrc_contact(pSrc, 99, NULL) == NULL, "past the contacts"); - CHECK(strncmp(DasSrc_path(pSrc), "tag:das2.org,2012:local:/", 25) == 0, "no path given, so named by location: %s", - DasSrc_path(pSrc)); - CHECK(strstr(DasSrc_path(pSrc), "/" HSK) != NULL, "the location is in the name"); + CHECK(streq(DasSrc_path(pSrc), ""), "no path given, so no URI: %s", DasSrc_path(pSrc)); /* Every property has a key, every control a parameter, all titled */ bool bAllKeyed = true, bAllParam = true, bAllTitled = true; @@ -682,7 +681,7 @@ static void test_juno(void) static void test_req(void) { - printf("Requisitions on their own:\n"); + printf("Requisitions without a source:\n"); DasReq* pReq = new_DasReq(); CHECK(pReq != NULL, "new"); CHECK(DasReq_count(pReq) == 0 && streq(DasReq_srcPath(pReq), ""), "empty and unstamped"); @@ -748,12 +747,14 @@ static void test_req(void) static void test_node(void) { printf("From a node tree:\n"); - DasNode* pRoot = new_DasNode_url(GEOMAG, "tag:das2.org,2012:site:/uiowa/geomag/flex", NULL, "TestSource"); + DasCat* pCat = new_DasCat(NULL, "TestSource"); + DasCat_setSeed(pCat, GEOMAG, "tag:das2.org,2012:site:/uiowa/geomag/flex"); + DasNode* pRoot = DasCat_seed(pCat); CHECK(pRoot != NULL, "node loads"); - if(pRoot == NULL) return; + if(pRoot == NULL){ del_DasCat(pCat); return; } DasSrc* pSrc = new_DasSrc_node(pRoot); CHECK(pSrc != NULL, "source from node"); - del_DasNode(pRoot); /* the source must not care */ + del_DasCat(pCat); /* takes the node; the source must not care */ if(pSrc){ CHECK(streq(DasSrc_path(pSrc), "tag:das2.org,2012:site:/uiowa/geomag/flex"), "path from the node"); CHECK(ctrlCount(pSrc, -1) == 17, "same interface after the tree is gone"); @@ -782,8 +783,7 @@ static void test_node(void) daslog_setlevel(DASLOG_WARN); CHECK(pSrc2 != NULL, "builds"); if(pSrc2){ - CHECK(streq(DasSrc_path(pSrc2), "tag:das2.org,2012:local:/memory"), "nameless and placeless: %s", - DasSrc_path(pSrc2)); + CHECK(streq(DasSrc_path(pSrc2), ""), "nameless and placeless means no URI: %s", DasSrc_path(pSrc2)); CHECK(DasSrc_propCount(pSrc2) == 3 && ctrlCount(pSrc2, -1) == 1, "three props, one control"); CHECK(streq(DasSrcProp_key(DasSrc_prop(pSrc2, 0)), "options.good"), "the good one"); CHECK(DasSrcProp_type(DasSrc_prop(pSrc2, 0)) == das_srcprop_bool, "boolean by default value"); @@ -800,9 +800,11 @@ static void test_pick(void) { printf("Loadable sources under a container:\n"); const char* aNames[4]; - DasNode* pTwin = new_DasNode_url("test/das_fetch/cat/twin.json", "site:/uiowa/twin", NULL, "TestSource"); + DasCat* pCat = new_DasCat(NULL, "TestSource"); + DasCat_setSeed(pCat, "test/das_fetch/cat/twin.json", "site:/uiowa/twin"); + DasNode* pTwin = DasCat_seed(pCat); CHECK(pTwin != NULL, "twin loads"); - if(pTwin == NULL) return; + if(pTwin == NULL){ del_DasCat(pCat); return; } CHECK(dassrc_loadable(pTwin, aNames, 4) == 2 && streq(aNames[0], "flex") && streq(aNames[1], "das3"), "two loadable children in catalog order, the DSDF is skipped"); CHECK(dassrc_loadable(pTwin, NULL, 0) == 2, "counting only"); @@ -810,17 +812,19 @@ static void test_pick(void) CHECK(new_DasSrc_node(pTwin) == NULL, "a container is not a source, no child is picked"); daslog_setlevel(DASLOG_WARN); - DasNode* pSub = DasNode_subNode(pTwin, "das3", NULL, NULL); + DasNode* pSub = DasCat_subNode(pCat, pTwin, "das3"); CHECK(pSub != NULL, "the caller's choice loads through the tree"); CHECK(dassrc_loadable(pSub, NULL, 0) == 0, "a source has no loadable children"); DasSrc* pSrc = (pSub == NULL) ? NULL : new_DasSrc_node(pSub); CHECK(pSrc != NULL && streq(DasSrc_path(pSrc), "tag:das2.org,2012:site:/uiowa/twin/das3"), "source from the chosen child"); del_DasSrc(pSrc); - del_DasNode(pTwin); + del_DasCat(pCat); - DasNode* pOld = new_DasNode_url("test/das_fetch/cat/old.json", "site:/uiowa/old", NULL, "TestSource"); + pCat = new_DasCat(NULL, "TestSource"); + DasCat_setSeed(pCat, "test/das_fetch/cat/old.json", "site:/uiowa/old"); + DasNode* pOld = DasCat_seed(pCat); CHECK(pOld != NULL, "old loads"); - if(pOld == NULL) return; + if(pOld == NULL){ del_DasCat(pCat); return; } CHECK(dassrc_loadable(pOld, aNames, 4) == 2, "a 2018 Collection lists both HttpStreamSrc children"); /* Presented in the current layout: the provides list each convention @@ -836,7 +840,7 @@ static void test_pick(void) char* sDoc = (char*)DasJdo_writeMinified(DasNode_getJdo(pOld, NULL), &uDoc); CHECK(sDoc != NULL && strstr(sDoc, "\"provides\":[\"application/vnd.hapi.stream\"]") != NULL, "the added list serializes"); free(sDoc); - del_DasNode(pOld); + del_DasCat(pCat); } int main(int argc, char** argv) diff --git a/test/TestWalk.c b/test/TestWalk.c new file mode 100644 index 0000000..1340763 --- /dev/null +++ b/test/TestWalk.c @@ -0,0 +1,413 @@ +/** @file TestWalk.c Unit tests for DasNodeWalk, driven by hand from + * fixture files. No network, no DasCat. */ + +/* Author: Chris Piker , via Claude Fable 5.1 + * + * This file is intended to demonstrate an interface. This is free + * and unencumbered software released into the public domain + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this file, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this file dedicate any and all copyright interest in this file to + * the public domain. We make this dedication for the benefit of the + * public at large and to the detriment of our heirs and successors. We + * intend this dedication to be an overt act of relinquishment in + * perpetuity of all present and future rights to this file under + * copyright law. + * + * THIS FILE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * + * For more information, please refer to + */ + +/* Runs from the top of the source tree, the fixture node files in + * test/das_walk/ refer to each other by paths from there. This program + * is the walk's driver, which is what a browser build is: it reads the + * file a walk asks for and feeds it back, or reports that it could not. + * Because the driver is in the test, the fallback paths that only a + * misbehaving server would exercise are exercised here on purpose: a + * missing mirror, a look-alike child, a document that is not JSON, a + * child with nowhere to load from. */ + +#include +#include +#include +#include + +#include +#include + +static int g_nPass = 0; +static int g_nFail = 0; + +#define CHECK(COND, ...) do{ \ + if(COND){ ++g_nPass; } \ + else{ ++g_nFail; printf(" FAIL %s:%d: ", __FILE__, __LINE__); printf(__VA_ARGS__); putchar('\n'); } \ +}while(0) + +static bool streq(const char* a, const char* b){ return (a != NULL) && (b != NULL) && (strcmp(a, b) == 0); } + +#define ROOT_URL "test/das_walk/root.json" +#define ROOT_URI "tag:das2.org,2012:site:/walk" + +/* ************************************************************************* */ +/* The driver: files in, documents out */ + +static char* readFile(const char* sPath, size_t* pLen) +{ + FILE* pFile = fopen(sPath, "rb"); + if(pFile == NULL) return NULL; + char* pBuf = NULL; + size_t uCap = 0, uLen = 0; + char sChunk[4096]; + size_t uRead; + while((uRead = fread(sChunk, 1, sizeof(sChunk), pFile)) > 0){ + if(uLen + uRead > uCap){ + uCap = (uLen + uRead) * 2; + pBuf = (char*)realloc(pBuf, uCap); + } + memcpy(pBuf + uLen, sChunk, uRead); + uLen += uRead; + } + fclose(pFile); + *pLen = uLen; + return pBuf; +} + +/* Records every URL asked for, in order, so a test can say what a walk + * should have tried */ +#define MAX_LOG 16 +struct drive_log { + const char* aUrl[MAX_LOG]; + size_t uUrls; +}; + +/* Answer one step with candidate uIdx: feed the file or report it missing */ +static DasErrCode step(DasNodeWalk* pWalk, size_t uIdx, struct drive_log* pLog) +{ + const char* sUrl = DasNodeWalk_url(pWalk, uIdx); + if((pLog != NULL) && (pLog->uUrls < MAX_LOG)) pLog->aUrl[pLog->uUrls++] = sUrl; + size_t uLen = 0; + char* pDoc = readFile(sUrl, &uLen); + if(pDoc == NULL){ + DasNodeWalk_fail(pWalk, uIdx, "no such file"); + return DASERR_NOTIMP; /* any code that is not DAS_OKAY or DASERR_NODE */ + } + DasErrCode rc = DasNodeWalk_feed(pWalk, uIdx, pDoc, uLen, NULL); + free(pDoc); + return rc; +} + +static DasNode* drive(DasNodeWalk* pWalk, struct drive_log* pLog) +{ + if(pLog != NULL) pLog->uUrls = 0; + while(DasNodeWalk_state(pWalk) == das_walk_fetch) step(pWalk, 0, pLog); + return DasNodeWalk_result(pWalk); +} + +static bool logIs(const struct drive_log* pLog, size_t u, ...) +{ + if(pLog->uUrls != u) return false; + va_list ap; + va_start(ap, u); + bool bOk = true; + for(size_t i = 0; i < u; ++i){ + const char* s = va_arg(ap, const char*); + if(!streq(pLog->aUrl[i], s)) bOk = false; + } + va_end(ap); + return bOk; +} + +static void showLog(const struct drive_log* pLog) +{ + printf(" asked for:"); + for(size_t i = 0; i < pLog->uUrls; ++i) printf(" %s", pLog->aUrl[i]); + putchar('\n'); +} + +static DasNode* loadRoot(void) +{ + size_t uLen = 0; + char* pDoc = readFile(ROOT_URL, &uLen); + DasNode* pRoot = (pDoc == NULL) ? NULL : new_DasNode_json(pDoc, uLen, ROOT_URL, "site:/walk"); + free(pDoc); + return pRoot; +} + +/* ************************************************************************* */ +static void test_json(void) +{ + printf("TEST: nodes from bytes\n"); + DasNode* pRoot = loadRoot(); + CHECK(pRoot != NULL, "root loads from bytes"); + if(pRoot == NULL) return; + CHECK(DasNode_isCatalog(pRoot), "it is a catalog"); + CHECK(streq(DasNode_pathUri(pRoot), ROOT_URI), "shorthand expanded: %s", DasNode_pathUri(pRoot)); + CHECK(streq(DasNode_srcUrl(pRoot), ROOT_URL), "source URL kept"); + CHECK(DasNode_subCount(pRoot) == 5, "five children listed, none loaded"); + CHECK(DasNode_subType(pRoot, 4) == d2node_stream_src, "last child is a stream source"); + del_DasNode(pRoot); + + daslog_setlevel(DASLOG_CRIT); + CHECK(new_DasNode_json("not json", 8, "x", NULL) == NULL, "garbage is not a node"); + CHECK(new_DasNode_json("{\"type\":\"Nope\"}", 15, "x", NULL) == NULL, "unknown type is not a node"); + daslog_setlevel(DASLOG_WARN); + + const char* sDoc = "{\"type\":\"HttpStreamSrc\",\"protocol\":{}}"; + DasNode* pNode = new_DasNode_json(sDoc, strlen(sDoc), NULL, NULL); + CHECK(pNode && (DasNode_pathUri(pNode) == NULL), "nameless means no URI, a root"); + CHECK(pNode && streq(DasNode_srcUrl(pNode), ""), "no URL is the empty string"); + del_DasNode(pNode); + pNode = new_DasNode_json(sDoc, strlen(sDoc), "http://host/x.json", NULL); + CHECK(pNode && (DasNode_pathUri(pNode) == NULL), "a location is not a name either"); + CHECK(pNode && streq(DasNode_srcUrl(pNode), "http://host/x.json"), "but it is kept as the source URL"); + del_DasNode(pNode); +} + +/* ************************************************************************* */ +static void test_steps(void) +{ + printf("TEST: one step at a time\n"); + DasNode* pRoot = loadRoot(); + if(pRoot == NULL) return; + + DasNodeWalk* pWalk = new_DasNodeWalk(pRoot, "src"); + CHECK(pWalk != NULL, "walk starts"); + if(pWalk == NULL){ del_DasNode(pRoot); return; } + CHECK(DasNodeWalk_state(pWalk) == das_walk_fetch, "a document is wanted"); + CHECK(DasNodeWalk_urlCount(pWalk) == 1, "one candidate"); + CHECK(streq(DasNodeWalk_url(pWalk, 0), "test/das_walk/src.json"), "the child's URL"); + CHECK(DasNodeWalk_url(pWalk, 1) == NULL, "past the end is NULL"); + CHECK(streq(DasNodeWalk_uri(pWalk), ROOT_URI "/src"), "the child's URI: %s", DasNodeWalk_uri(pWalk)); + CHECK(DasNodeWalk_result(pWalk) == NULL, "no result yet"); + CHECK(DasNodeWalk_nearest(pWalk, NULL) == NULL, "nothing nearest while running"); + + daslog_setlevel(DASLOG_CRIT); + CHECK(DasNodeWalk_feed(pWalk, 1, "{}", 2, NULL) != DAS_OKAY, "feeding a candidate not on offer is refused"); + daslog_setlevel(DASLOG_WARN); + CHECK(DasNodeWalk_state(pWalk) == das_walk_fetch, "and changes nothing"); + + CHECK(step(pWalk, 0, NULL) == DAS_OKAY, "the source document parses"); + CHECK(DasNodeWalk_state(pWalk) == das_walk_done, "walk is done"); + DasNode* pSrc = DasNodeWalk_result(pWalk); + CHECK(pSrc != NULL, "result"); + CHECK(pSrc && DasNode_isStreamSrc(pSrc), "it is a stream source"); + CHECK(pSrc && streq(DasNode_pathUri(pSrc), ROOT_URI "/src"), "named by the walk"); + CHECK(pSrc && streq(DasNode_srcUrl(pSrc), "test/das_walk/src.json"), "loaded from the candidate"); + CHECK(DasNodeWalk_urlCount(pWalk) == 0, "nothing on offer when done"); + daslog_setlevel(DASLOG_CRIT); + CHECK(DasNodeWalk_feed(pWalk, 0, "{}", 2, NULL) != DAS_OKAY, "a finished walk takes nothing"); + daslog_setlevel(DASLOG_WARN); + del_DasNodeWalk(pWalk); + + /* The result was attached: the same path needs no document now */ + pWalk = new_DasNodeWalk(pRoot, "src"); + CHECK(pWalk && (DasNodeWalk_state(pWalk) == das_walk_done), "second walk is done at once"); + CHECK(pWalk && (DasNodeWalk_result(pWalk) == pSrc), "and finds the same node"); + del_DasNodeWalk(pWalk); + pWalk = new_DasNodeWalk(pRoot, "/src"); + CHECK(pWalk && (DasNodeWalk_result(pWalk) == pSrc), "leading separator is accepted"); + del_DasNodeWalk(pWalk); + pWalk = new_DasNodeWalk(pRoot, ""); + CHECK(pWalk && (DasNodeWalk_result(pWalk) == pRoot), "empty path is the start node"); + del_DasNodeWalk(pWalk); + + daslog_setlevel(DASLOG_CRIT); + CHECK(new_DasNodeWalk(pSrc, "deeper") == NULL, "no walk below a source"); + daslog_setlevel(DASLOG_WARN); + + del_DasNode(pSrc); /* a child: ignored, its parent owns it */ + CHECK(streq(DasNode_srcUrl(pSrc), "test/das_walk/src.json"), "a child is not deleted separately"); + del_DasNode(pRoot); +} + +/* ************************************************************************* */ +static void test_fallback(void) +{ + printf("TEST: mirrors, look-alikes and dead ends\n"); + struct drive_log log; + DasNode* pRoot = loadRoot(); + if(pRoot == NULL) return; + + /* The first mirror of "a" does not exist */ + DasNodeWalk* pWalk = new_DasNodeWalk(pRoot, "a/b"); + CHECK(pWalk && (DasNodeWalk_urlCount(pWalk) == 2), "two mirrors on offer"); + DasNode* pB = drive(pWalk, &log); + CHECK(pB != NULL, "a/b resolves through the second mirror"); + CHECK(logIs(&log, 3, "test/das_walk/missing.json", "test/das_walk/a.json", "test/das_walk/src.json"), + "missing mirror, good mirror, leaf"); + if(g_nFail) showLog(&log); + CHECK(pB && streq(DasNode_pathUri(pB), ROOT_URI "/a/b"), "URI composed through two levels: %s", + pB ? DasNode_pathUri(pB) : "(null)"); + del_DasNodeWalk(pWalk); + + /* Both levels attached: "a" costs nothing now, and so does a/b */ + pWalk = new_DasNodeWalk(pRoot, "a"); + CHECK(pWalk && (DasNodeWalk_state(pWalk) == das_walk_done), "the intermediate node was kept"); + DasNode* pA = pWalk ? DasNodeWalk_result(pWalk) : NULL; + del_DasNodeWalk(pWalk); + pWalk = new_DasNodeWalk(pRoot, "a/b"); + CHECK(pWalk && (DasNodeWalk_result(pWalk) == pB), "and so was the leaf"); + del_DasNodeWalk(pWalk); + pWalk = new_DasNodeWalk(pA, "b"); + CHECK(pWalk && (DasNodeWalk_result(pWalk) == pB), "a walk may start at the intermediate node"); + del_DasNodeWalk(pWalk); + + /* "a" is a prefix of "ab", so it is tried first and rejected by the + separator rule; the loaded copy of "a" is consulted and the listing + entry for it skipped, so no mirror of "a" is asked for again */ + pWalk = new_DasNodeWalk(pRoot, "ab/x"); + DasNode* pX = drive(pWalk, &log); + CHECK(pX != NULL, "ab/x resolves"); + CHECK(logIs(&log, 2, "test/das_walk/ab.json", "test/das_walk/src.json"), "look-alike skipped via the loaded copy"); + if(g_nFail) showLog(&log); + del_DasNodeWalk(pWalk); + del_DasNode(pRoot); + + /* The same on a fresh tree: "a" has to be fetched to be rejected, and + since it led nowhere it is not kept */ + pRoot = loadRoot(); + pWalk = new_DasNodeWalk(pRoot, "ab/x"); + pX = drive(pWalk, &log); + CHECK(pX && streq(DasNode_pathUri(pX), ROOT_URI "/ab/x"), "ab/x on a fresh tree"); + CHECK(logIs(&log, 4, "test/das_walk/missing.json", "test/das_walk/a.json", "test/das_walk/ab.json", + "test/das_walk/src.json"), "a was tried first, through both mirrors, then ab"); + if(g_nFail) showLog(&log); + del_DasNodeWalk(pWalk); + pWalk = new_DasNodeWalk(pRoot, "a"); + CHECK(pWalk && (DasNodeWalk_state(pWalk) == das_walk_fetch), "a dead end is not kept"); + del_DasNodeWalk(pWalk); + + /* A document that is not JSON: feed says so, the walk moves on */ + pWalk = new_DasNodeWalk(pRoot, "junk/x"); + daslog_setlevel(DASLOG_CRIT); + DasErrCode rc = step(pWalk, 0, NULL); + daslog_setlevel(DASLOG_WARN); + CHECK(rc == DASERR_NODE, "junk is reported to the driver"); + CHECK(DasNodeWalk_state(pWalk) == das_walk_failed, "and nothing else to try"); + const char* sRest = NULL; + CHECK(streq(DasNodeWalk_nearest(pWalk, &sRest), ROOT_URI) && streq(sRest, "junk/x"), + "nearest is the root with the whole path below it"); + del_DasNodeWalk(pWalk); + + /* Nothing listed, or listed with no URLs: failed before any step */ + daslog_setlevel(DASLOG_CRIT); + pWalk = new_DasNodeWalk(pRoot, "nothere"); + CHECK(pWalk && (DasNodeWalk_state(pWalk) == das_walk_failed), "unknown child fails at once"); + CHECK(pWalk && (DasNodeWalk_urlCount(pWalk) == 0), "nothing on offer"); + del_DasNodeWalk(pWalk); + pWalk = new_DasNodeWalk(pRoot, "nourls/x"); + CHECK(pWalk && (DasNodeWalk_state(pWalk) == das_walk_failed), "a child with no URLs fails at once"); + del_DasNodeWalk(pWalk); + daslog_setlevel(DASLOG_WARN); + + /* A driver that fetched the mirrors in parallel feeds the one that + answered, skipping the ones before it */ + pWalk = new_DasNodeWalk(pRoot, "a/b"); + CHECK(pWalk && (DasNodeWalk_urlCount(pWalk) == 2), "two mirrors again on a fresh tree"); + CHECK(step(pWalk, 1, NULL) == DAS_OKAY, "the second mirror fed directly"); + CHECK(DasNodeWalk_state(pWalk) == das_walk_fetch, "on to the leaf"); + CHECK(streq(DasNodeWalk_uri(pWalk), ROOT_URI "/a/b"), "leaf URI"); + /* Abandon it here: the node loaded for "a" belongs to the walk and + must go with it, see valgrind */ + del_DasNodeWalk(pWalk); + pWalk = new_DasNodeWalk(pRoot, "a"); + CHECK(pWalk && (DasNodeWalk_state(pWalk) == das_walk_fetch), "an abandoned walk attached nothing"); + del_DasNodeWalk(pWalk); + + del_DasNode(pRoot); +} + +/* ************************************************************************* */ +static void test_federated(void) +{ + printf("TEST: the compiled in seed\n"); + size_t uRoots = 0; + const char** aRoots = das_seed_urls(&uRoots); + + daslog_setlevel(DASLOG_CRIT); + CHECK(new_DasNodeWalk(NULL, "src") == NULL, "a walk needs a start node"); + daslog_setlevel(DASLOG_WARN); + + DasNodeWalk* pWalk = new_DasNodeWalk_seed(); + CHECK(pWalk != NULL, "seed walk starts"); + if(pWalk == NULL) return; + CHECK(DasNodeWalk_state(pWalk) == das_walk_fetch, "a seed document is wanted"); + CHECK(DasNodeWalk_urlCount(pWalk) == uRoots, "every seed URL is a candidate"); + CHECK(streq(DasNodeWalk_url(pWalk, 0), aRoots[0]), "in order"); + CHECK(streq(DasNodeWalk_uri(pWalk), ""), "a seed has no URI"); + + DasNodeWalk_fail(pWalk, 0, "no network in a unit test"); + CHECK(DasNodeWalk_urlCount(pWalk) == uRoots - 1, "one mirror fewer"); + CHECK(streq(DasNodeWalk_url(pWalk, 0), aRoots[1]), "the next mirror is first now"); + + /* A seed must be a Catalog; a source document is dropped */ + size_t uLen = 0; + char* pDoc = readFile("test/das_walk/src.json", &uLen); + daslog_setlevel(DASLOG_CRIT); + DasErrCode rc = DasNodeWalk_feed(pWalk, 0, pDoc, uLen, NULL); + daslog_setlevel(DASLOG_WARN); + free(pDoc); + CHECK(rc == DASERR_NODE, "a source is not a seed"); + CHECK(DasNodeWalk_state(pWalk) == das_walk_failed, "no mirrors left"); + CHECK(DasNodeWalk_nearest(pWalk, NULL) == NULL, "nothing nearest when no seed loaded"); + del_DasNodeWalk(pWalk); + + /* A stand in seed that lists the fixture tree as the site: node, so + its children are site:/a, site:/src and so on. The seed is the + caller's; everything else is a walk from it and stays under it. */ + pWalk = new_DasNodeWalk_seed(); + pDoc = readFile("test/das_walk/fedroot.json", &uLen); + CHECK(DasNodeWalk_feed(pWalk, 0, pDoc, uLen, "https://mirror/index.json") == DAS_OKAY, "a stand in seed"); + free(pDoc); + CHECK(DasNodeWalk_state(pWalk) == das_walk_done, "the seed is the result"); + DasNode* pRoot = DasNodeWalk_result(pWalk); + del_DasNodeWalk(pWalk); + CHECK(pRoot && DasNode_isCatalog(pRoot), "the seed outlives the walk"); + CHECK(pRoot && (DasNode_pathUri(pRoot) == NULL), "a seed with no prefix is a root, the one node with no URI"); + CHECK(pRoot && streq(DasNode_srcUrl(pRoot), aRoots[0]), "loaded from the first mirror, this walk never failed one"); + if(pRoot == NULL) return; + + pWalk = new_DasNodeWalk(pRoot, "tag:das2.org,2012:site:/src"); + CHECK(pWalk && (DasNodeWalk_state(pWalk) == das_walk_fetch), "a full URI walks from the seed"); + CHECK(pWalk && streq(DasNodeWalk_uri(pWalk), "tag:das2.org,2012:site:"), "first level named by the seed's prefix child: %s", + pWalk ? DasNodeWalk_uri(pWalk) : "(null)"); + CHECK(pWalk && streq(DasNodeWalk_url(pWalk, 0), ROOT_URL), "the fixture root stands in for site"); + DasNode* pSrc = pWalk ? drive(pWalk, NULL) : NULL; + CHECK(pSrc && streq(DasNode_pathUri(pSrc), "tag:das2.org,2012:site:/src"), "full URI: %s", + pSrc ? DasNode_pathUri(pSrc) : "(null)"); + del_DasNodeWalk(pWalk); + + /* A sibling shares the levels already walked */ + struct drive_log log; + pWalk = new_DasNodeWalk(pRoot, "tag:das2.org,2012:site:/a/b"); + DasNode* pB = pWalk ? drive(pWalk, &log) : NULL; + CHECK(pB != NULL, "a sibling path resolves"); + CHECK(logIs(&log, 3, "test/das_walk/missing.json", "test/das_walk/a.json", "test/das_walk/src.json"), + "the site level was not fetched again"); + if(g_nFail) showLog(&log); + del_DasNodeWalk(pWalk); + + del_DasNode(pRoot); /* takes site, a, b and src with it */ +} + +/* ************************************************************************* */ +int main(int argc, char** argv) +{ + das_init(argv[0], DASERR_DIS_EXIT, 0, DASLOG_WARN, NULL); + + test_json(); + test_steps(); + test_fallback(); + test_federated(); + + printf("%d checks passed, %d failed\n", g_nPass, g_nFail); + return (g_nFail == 0) ? 0 : 1; +} diff --git a/test/das_fetch_test.sh b/test/das_fetch_test.sh index bc878fb..cbdcd39 100755 --- a/test/das_fetch_test.sh +++ b/test/das_fetch_test.sh @@ -93,46 +93,54 @@ expect_url $G examples.0 -- \ expect_url $G -- \ "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" -# -r names the root node, URI=URL, and PATH is then optional. A bare -# location is refused: a node does not know its own name, the user says -# what the file stands for. +# -r names the seed node, URL=URI, and PATH is then optional. A bare +# location is a seed with no name, a root, which works too; the old +# URI=URL order is refused with a hint. SU="tag:das2.org,2012:site:/uiowa/tracers/tracers-2/spice/geomagcoords/flex" HU="tag:das2.org,2012:site:/uiowa/tracers/tracers-2/l1a/cdpu/hsk/flex" -expect_rc 2 url -r $G 0 # a root needs a name -expect_rc 2 url -r =$G 0 # still no name -expect_url -r $SU=$G 0 -- \ +expect_url -r $G 0 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" # a root seed +expect_rc 2 url -r =$G 0 # nothing before the '=' +expect_rc 2 url -r $SU=$G 0 # the old order, URI=URL +expect_url -r $G=$SU 0 -- \ "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" -expect_url -r $SU=$G $SU 0 -- \ +expect_url -r $G=$SU $SU 0 -- \ "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" -if $FETCH node -r $SU=$G 2>/dev/null | grep -q "^URI: $SU\$"; then +if $FETCH node -r $G=$SU 2>/dev/null | grep -q "^URI: $SU\$"; then nPass=$((nPass + 1)) else - echo " FAIL: das_fetch node -r $SU=$G did not report the root URI"; nFail=$((nFail + 1)) + echo " FAIL: das_fetch node -r $G=$SU did not report the seed URI"; nFail=$((nFail + 1)) fi # -m mounts the same node on the federated catalog. Offline only the -# mounted side can be exercised; -m and -r URL together is a usage error. -expect_url -m $SU=$G 0 -- \ +# mounted side can be exercised, and a mount is not a place to start, so +# the path is named; -m and -r URL together is a usage error. +expect_url -m $G=$SU $SU 0 -- \ "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" -export DASFETCH_MOUNT=$SU=$G -expect_url 0 -- \ +expect_rc 2 url -m $G=$SU 0 # a mount gives no default path +export DASFETCH_MOUNT=$G=$SU +expect_url $SU 0 -- \ "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" unset DASFETCH_MOUNT -expect_url -m $SU=$G -r $HU=$H 0 -- \ - "$HB?read.time.min=2026-01-06T14%3A12&read.time.max=2026-01-06T14%3A24" # a root and a mount make an offline catalog -expect_url -m $SU=$G -r $HU=$H $SU 0 -- \ +expect_url -m $G=$SU -r $H=$HU 0 -- \ + "$HB?read.time.min=2026-01-06T14%3A12&read.time.max=2026-01-06T14%3A24" # a seed and a mount make an offline catalog +expect_url -m $G=$SU -r $H=$HU $SU 0 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" +expect_rc 2 url -m site:/uiowa 0 # mount needs URL=URI +# Under a root seed a mount is named by a path from the root, and it is +# consulted for the relative spelling, shadowing the listing +expect_url -r $FX/cat/site.json -m $G=uiowa/broken uiowa/broken 0 -- \ "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" -expect_rc 2 url -m site:/uiowa 0 # mount needs URI=URL -expect_rc 2 url -r site:/uiowa=nourl 0 # '=' but no URL after it -expect_rc 3 url -r $SU=$G site:/uiowa/juno # a named root is the whole catalog +expect_rc 2 url -r nourl=site:/uiowa 0 # '=' but no URL before it +expect_rc 3 url -r $G=$SU site:/uiowa/juno # a seed is the whole catalog -# DASFETCH_ROOT stands in for -r, and -r still wins over it -export DASFETCH_ROOT=$SU=$G +# DASFETCH_START stands in for -r, and -r still wins over it +export DASFETCH_START=$G=$SU expect_url 0 -- \ "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" -expect_url -r $HU=$H 0 -- \ +expect_url -r $H=$HU 0 -- \ "$HB?read.time.min=2026-01-06T14%3A12&read.time.max=2026-01-06T14%3A24" -unset DASFETCH_ROOT +unset DASFETCH_START # dotted member paths, section names may be plural or singular expect_url $G 0 data.lshell coord.time.inter=30 -- \ @@ -173,33 +181,33 @@ expect_url $J 7 resolution=10 -- \ # no -a das_fetch asks for das3 streams, then anything. An -a list is a set. T=$FX/cat/twin.json O=$FX/cat/old.json -expect_url -r site:/uiowa/twin=$T 0 -- \ +expect_url -r $T=site:/uiowa/twin 0 -- \ "$HB?read.time.min=2026-01-06T14%3A12&read.time.max=2026-01-06T14%3A24" -expect_url -r site:/uiowa/twin=$T -a text/csv 0 -- \ +expect_url -r $T=site:/uiowa/twin -a text/csv 0 -- \ "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" -expect_url -r site:/uiowa/twin=$T -a 'image/png, application/x-cdf' 0 -- \ +expect_url -r $T=site:/uiowa/twin -a 'image/png, application/x-cdf' 0 -- \ "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" -expect_rc 3 url -r site:/uiowa/twin=$T -a '*/*' 0 # two sources qualify, name one -expect_rc 3 url -r site:/uiowa/twin=$T -a 'application/vnd.das.stream, text/csv' 0 # order does not pick -expect_rc 3 url -r site:/uiowa/twin=$T -a image/png 0 # none does -expect_url -r site:/uiowa/twin=$T site:/uiowa/twin/flex -a '*/*' 0 -- \ +expect_rc 3 url -r $T=site:/uiowa/twin -a '*/*' 0 # two sources qualify, name one +expect_rc 3 url -r $T=site:/uiowa/twin -a 'application/vnd.das.stream, text/csv' 0 # order does not pick +expect_rc 3 url -r $T=site:/uiowa/twin -a image/png 0 # none does +expect_url -r $T=site:/uiowa/twin site:/uiowa/twin/flex -a '*/*' 0 -- \ "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" -if $FETCH node -r site:/uiowa/twin=$T -a '*/*' 2>/dev/null | grep -q "^ das3 "; then +if $FETCH node -r $T=site:/uiowa/twin -a '*/*' 2>/dev/null | grep -q "^ das3 "; then nPass=$((nPass + 1)) else echo " FAIL: das_fetch node with an ambiguous pick did not list the children"; nFail=$((nFail + 1)) fi # a 2018 Collection has no provides lists, convention stands in -expect_url -r site:/uiowa/old=$O -a application/vnd.das2.das2stream time=2021-02-21,2021-02-22 -- \ +expect_url -r $O=site:/uiowa/old -a application/vnd.das2.das2stream time=2021-02-21,2021-02-22 -- \ "$JB&end_time=2021-02-22&start_time=2021-02-21" -expect_rc 3 url -r site:/uiowa/old=$O time=2021-02-21,2021-02-22 # nothing provides das3, so anything: both qualify -expect_rc 3 url -r site:/uiowa/old=$O -a text/csv time=2021-02-21,2021-02-22 +expect_rc 3 url -r $O=site:/uiowa/old time=2021-02-21,2021-02-22 # nothing provides das3, so anything: both qualify +expect_rc 3 url -r $O=site:/uiowa/old -a text/csv time=2021-02-21,2021-02-22 # -m may be repeated, and the deeper mount wins whatever the order -expect_url -m site:/uiowa/twin=$T -m site:/uiowa/twin/flex=$G -a '*/*' site:/uiowa/twin/flex 0 -- \ +expect_url -m $T=site:/uiowa/twin -m $G=site:/uiowa/twin/flex -a '*/*' site:/uiowa/twin/flex 0 -- \ "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" -expect_url -m site:/uiowa/twin/flex=$G -m site:/uiowa/twin=$T -a '*/*' site:/uiowa/twin/flex 0 -- \ +expect_url -m $G=site:/uiowa/twin/flex -m $T=site:/uiowa/twin -a '*/*' site:/uiowa/twin/flex 0 -- \ "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" # --- error paths, by exit code ---------------------------------------------- diff --git a/test/das_walk/a.json b/test/das_walk/a.json new file mode 100644 index 0000000..df77d68 --- /dev/null +++ b/test/das_walk/a.json @@ -0,0 +1,9 @@ +{ + "type": "Catalog", + "version": "0.5", + "name": "a", + "separator": "/", + "catalog": { + "b": {"type": "HttpStreamSrc", "urls": ["test/das_walk/src.json"]} + } +} diff --git a/test/das_walk/ab.json b/test/das_walk/ab.json new file mode 100644 index 0000000..a4f9f4a --- /dev/null +++ b/test/das_walk/ab.json @@ -0,0 +1,9 @@ +{ + "type": "Catalog", + "version": "0.5", + "name": "ab", + "separator": "/", + "catalog": { + "x": {"type": "HttpStreamSrc", "urls": ["test/das_walk/src.json"]} + } +} diff --git a/test/das_walk/fedroot.json b/test/das_walk/fedroot.json new file mode 100644 index 0000000..4640f14 --- /dev/null +++ b/test/das_walk/fedroot.json @@ -0,0 +1,10 @@ +{ + "type": "Catalog", + "version": "0.5", + "name": "", + "title": "Stands in for the federated root: children are named by URI prefix", + "separator": null, + "catalog": { + "tag:das2.org,2012:site:": {"type": "Catalog", "urls": ["test/das_walk/root.json"]} + } +} diff --git a/test/das_walk/junk.json b/test/das_walk/junk.json new file mode 100644 index 0000000..85e4783 --- /dev/null +++ b/test/das_walk/junk.json @@ -0,0 +1 @@ +this is not a node document diff --git a/test/das_walk/root.json b/test/das_walk/root.json new file mode 100644 index 0000000..b8eacbd --- /dev/null +++ b/test/das_walk/root.json @@ -0,0 +1,33 @@ +{ + "type": "Catalog", + "version": "0.5", + "name": "root", + "title": "Walk test root: mirrors, look-alikes, junk and a dead listing", + "separator": "/", + "catalog": { + "a": { + "type": "Catalog", + "title": "First mirror is missing, second works", + "urls": ["test/das_walk/missing.json", "test/das_walk/a.json"] + }, + "ab": { + "type": "Catalog", + "title": "Its name starts with a, so a is tried first for ab/x", + "urls": ["test/das_walk/ab.json"] + }, + "junk": { + "type": "Catalog", + "title": "The file is not JSON", + "urls": ["test/das_walk/junk.json"] + }, + "nourls": { + "type": "Catalog", + "title": "Listed with nowhere to load from", + "urls": [] + }, + "src": { + "type": "HttpStreamSrc", + "urls": ["test/das_walk/src.json"] + } + } +} diff --git a/test/das_walk/src.json b/test/das_walk/src.json new file mode 100644 index 0000000..a47874f --- /dev/null +++ b/test/das_walk/src.json @@ -0,0 +1,8 @@ +{ + "type": "HttpStreamSrc", + "version": "0.7", + "name": "src", + "title": "The leaf every path in this fixture ends at", + "protocol": {"baseUrls": ["http://127.0.0.1:8765/walk"], "httpParams": {}}, + "interface": {} +} diff --git a/utilities/das_fetch.c b/utilities/das_fetch.c index 33ad4ae..c05749e 100644 --- a/utilities/das_fetch.c +++ b/utilities/das_fetch.c @@ -23,8 +23,8 @@ * job: DasCat (das3/catalog.h) resolves paths, DasSrc (das3/source.h) turns * a source node's "interface" section into keys and a DasReq (das3/req.h) * of KEY=VALUE settings into the HTTP GET URL the "protocol" section calls - * for. What lives here is the command line spelling of all that: the root - * policy options, the example shortcuts, and the listings. + * for. What lives here is the command line spelling of all that: the + * options for where paths start, the example shortcuts, and the listings. * * The body of a data response is never inspected. It may be a das2 or das3 * stream, a CDF file, CSV text, or CCSDS packets. Bytes are bytes. @@ -91,7 +91,7 @@ #define URL_SZ 4096 #define ERR_SZ 1024 -#define ROOT_ENV "DASFETCH_ROOT" +#define START_ENV "DASFETCH_START" #define MOUNT_ENV "DASFETCH_MOUNT" #define MAX_MOUNTS 16 #define MAX_KIDS 32 @@ -111,7 +111,7 @@ static void prnHelpMain(void) " " PROG " - fetch data by das catalog path, no server URLs required\n" "\n" "USAGE\n" -" " PROG " [options] roots\n" +" " PROG " [options] seeds\n" " " PROG " [options] node [PATH]\n" " " PROG " [options] json PATH[#FRAGMENT]\n" " " PROG " [options] url PATH [KEY=VALUE ...]\n" @@ -131,20 +131,20 @@ static void prnHelpMain(void) " source produces: das2 or das3 stream, CDF, CSV, or raw packets.\n" "\n" "COMMANDS\n" -" roots Print the compiled in catalog root URLs.\n" +" seeds Print the compiled in catalog seed URLs.\n" " node [PATH] List a catalog's children, or a data source's examples\n" -" and query keys. No PATH shows the default root.\n" +" and query keys. No PATH shows where paths start.\n" " json PATH Print a node's raw JSON, or a fragment of it.\n" " url PATH ... Print the HTTP GET URL a query turns into.\n" " data PATH ... Run the query, write the response body to stdout.\n" "\n" -" Each command has its own help: '" PROG " data -h' explains query keys,\n" +" Each command has a help page: '" PROG " data -h' explains query keys,\n" " '" PROG " node -h' explains the listings.\n" "\n" "PATHS\n" " Paths are catalog paths, not web paths. Any of these forms work:\n" "\n" -" uiowa/juno/wav/survey relative to the root prefix, by\n" +" uiowa/juno/wav/survey relative to the prefix, by\n" " default '" DASCAT_DEF_PREFIX "'\n" " site:/uiowa/juno/wav/survey shorthand for the URI form below\n" " tag:das2.org,2012:site:/uiowa full catalog path URI\n" @@ -155,28 +155,31 @@ static void prnHelpMain(void) " -h, --help This text, or with a command, that command's help.\n" " -V, --version Print the program and library version and exit.\n" "\n" -" -r, --root ROOT Where paths start, in one of two forms.\n" -" A catalog URI prefix for relative paths, default\n" +" -r, --start START Where relative paths start, in one of three forms.\n" +" A catalog URI prefix, default\n" " '" DASCAT_DEF_PREFIX "':\n" " -r tag:das2.org,2012:test:/\n" -" URI=URL, the node at URL declared to be the catalog\n" -" node URI. It becomes the only root: paths are\n" -" relative to it unless they are full URIs under it,\n" -" and nothing outside it can be reached. Any -m mounts\n" -" join it to make an offline catalog:\n" -" -r site:=./my_site_copy.json\n" -" -r 10.1037/a0040251=https://doi.org/10.1037/a0040251/root.json\n" -" A node has no name of its own, names are composed\n" -" walking down from the root, so the URI is required.\n" -" Without -r the environment variable " ROOT_ENV " is\n" +" A URL or file, the node there becomes the seed, a\n" +" root: paths are relative to it, names below it are\n" +" paths from it, and nothing outside it can be\n" +" reached:\n" +" -r ./my_site_copy.json\n" +" URL=URI, the same with the node declared to be the\n" +" catalog node URI, so that saved requisitions match\n" +" the node the file stands in for. Any -m mounts join\n" +" a seed to make an offline catalog:\n" +" -r ./my_site_copy.json=site:\n" +" -r https://doi.org/10.1037/a0040251/root.json=10.1037/a0040251\n" +" Without -r the environment variable " START_ENV " is\n" " used the same way, if set.\n" -" -m, --mount URI=URL\n" -" Mount a node on the federated catalog: the node at URL\n" -" stands in for URI and everything under it, the rest\n" -" of the catalog is unchanged. Relative paths start at\n" -" the mounted node, or at the -r prefix if one is given:\n" -" -m site:/uiowa/tracers/tracers-1=\\\n" -" https://tracers-dev.physics.uiowa.edu/stream/source/tracers-1.json\n" +" -m, --mount URL=NAME\n" +" Mount a node: the node at URL stands in for NAME and\n" +" everything under it, the rest of the catalog is\n" +" unchanged. NAME is a catalog URI, or with a -r URL\n" +" seed, a path from it. A mount does not move where\n" +" relative paths start:\n" +" -m https://tracers-dev.physics.uiowa.edu/stream/source/tracers-1.json=\\\n" +" site:/uiowa/tracers/tracers-1\n" " May be repeated. The environment variable " MOUNT_ENV "\n" " stands in for -m, if set.\n" "\n" @@ -234,11 +237,11 @@ static void prnHelpNode(void) "USAGE\n" " " PROG " [options] node [PATH]\n" "\n" -" Show what is at a catalog path. With no PATH, the default root, which\n" +" Show what is at a catalog path. With no PATH, where paths start, which\n" " lists the federated sites.\n" "\n" "CATALOGS\n" -" A catalog node prints its own name and URI, then one line per child:\n" +" A catalog node prints its name and URI, then one line per child:\n" " name, node type, and title. Append a child's name to the path to step\n" " down. The URI line ends with the separator this catalog uses, so the\n" " next path is URI plus child name. A '*' after the type marks node\n" @@ -278,7 +281,7 @@ static void prnHelpNode(void) " " PROG " node\n" " " PROG " node uiowa/juno/wav\n" " " PROG " node -s examples,coords uiowa/tracers/tracers-2/spice/geomagcoords\n" -" " PROG " -r site:/uiowa/tracers=https://tracers-dev.physics.uiowa.edu/stream/root.json node\n" +" " PROG " -r https://tracers-dev.physics.uiowa.edu/stream/root.json=site:/uiowa/tracers node\n" "\n", stdout); } @@ -361,7 +364,7 @@ static void prnHelpQuery(bool bData) "\n" " One housekeeping value from a development server, through das3_csv:\n" "\n" -" " PROG " -r site:/uiowa/tracers=https://tracers-dev.physics.uiowa.edu/stream/root.json \\\n" +" " PROG " -r https://tracers-dev.physics.uiowa.edu/stream/root.json=site:/uiowa/tracers \\\n" " data tracers-2/ql/mag/hsk time=2026-01-01,2026-01-02 sprt | das3_csv\n" "\n" " The URL a query turns into, without running it:\n" @@ -387,15 +390,15 @@ static void prnHelpJson(void) "\n", stdout); } -static void prnHelpRoots(void) +static void prnHelpSeeds(void) { fputs( "USAGE\n" -" " PROG " roots\n" +" " PROG " seeds\n" "\n" -" Print the compiled in URLs of the federated catalog root, one per line.\n" -" The root is mirrored, the first URL that answers is used. A -r ROOT\n" -" argument or the " ROOT_ENV " variable replaces it, a -m URI=URL argument\n" +" Print the compiled in URLs of the das catalog index, one per line.\n" +" The index is mirrored, the first URL that answers is used. A -r START\n" +" argument or the " START_ENV " variable replaces it, a -m URL=URI argument\n" " or the " MOUNT_ENV " variable mounts a node on it, see '" PROG " -h'.\n" "\n", stdout); } @@ -408,7 +411,7 @@ static void prnHelp(const char* sCmd) else if((strcmp(sCmd, "data") == 0)||(strcmp(sCmd, "get") == 0)) prnHelpQuery(true); else if(strcmp(sCmd, "url") == 0) prnHelpQuery(false); else if(strcmp(sCmd, "json") == 0) prnHelpJson(); - else if(strcmp(sCmd, "roots") == 0) prnHelpRoots(); + else if(strcmp(sCmd, "seeds") == 0) prnHelpSeeds(); else prnHelpMain(); } @@ -863,14 +866,19 @@ static void listCatalog(const DasNode* pNode, FILE* pOut, const char* sUri) } } -/* What to show as a node's name. Every node has a catalog URI, nodes - * loaded by location carry a local: one, see D2URI_LOCAL. Catalogs get +/* What to show as a node's name. A root has no URI: the caller's + * fallback is shown when it has one, the location the node was loaded + * from, else the node is called what it is. Catalogs with a URI get * their child separator appended so the next step is obvious. */ static void nodeUri( const DasNode* pNode, const char* sFallback, char* sBuf, size_t uLen ){ const char* sUri = DasNode_pathUri(pNode); - if((sUri == NULL)||(sUri[0] == '\0')) sUri = sFallback; + if((sUri == NULL)||(sUri[0] == '\0')){ + if((sFallback != NULL) && (sFallback[0] != '\0')) snprintf(sBuf, uLen, "(the root, %s)", sFallback); + else snprintf(sBuf, uLen, "(the root)"); + return; + } const char* sSep = DasNode_isCatalog(pNode) ? DasNode_pathSep(pNode) : NULL; snprintf(sBuf, uLen, "%s%s", sUri, sSep ? sSep : ""); } @@ -878,49 +886,61 @@ static void nodeUri( /* ************************************************************************* */ /* Catalog navigation * - * DasCat (das3/catalog.h) owns the roots, the credentials, the agent and the - * memory of every node loaded, and resolves paths under one root policy. - * What stays here is the command line spelling of that policy and the - * wording of a failed lookup. */ - - -/* A -r ROOT or -m URI=URL value, or their environment stand-ins, turned into - * catalog policy. URI=URL names a node: the whole catalog for -r, a mount - * on the federated catalog for -m. For -r anything else is a URI prefix. - * A location alone is refused for both: nodes do not know their own - * names, so a file offered as a root with no URI would get a local: name - * and its requisitions would never match the catalog node it copies. - * The URL side always has a scheme or looks like a file, which is how the - * '=' that matters is found when the URI itself contains one. Command line - * mistakes are reported here, the catalog logs its own refusals. False on - * either. */ -static bool applyRootArg(DasCat* pCat, const char* sVal, bool bMount) + * DasCat (das3/catalog.h) owns the seed, the mounts, the credentials, the + * agent and the memory of every node loaded, and resolves paths under one + * policy. What stays here is the command line spelling of that policy + * and the wording of a failed lookup. */ + + +/* A -r START or -m URL=URI value, or their environment stand-ins, turned + * into catalog policy. URL=URI names a node: the seed for -r, a mount for + * -m. For -r a URL alone is a seed with no name, a root, and anything + * else is a URI prefix. A mount always needs a name. + * + * The split is at the last '='. What follows it is a name when it has a + * scheme, "site:/x", or, since a mount under a root seed is named by a + * path, when it is not a location and the URL before it carries no query + * string. A URL with a query string can end in "=x" itself, so a mount + * on one needs a name with a scheme. Command line mistakes are reported + * here, the catalog logs its refusals. False on either. */ +static bool applyStartArg(DasCat* pCat, const char* sVal, bool bMount) { - for(const char* pEq = strchr(sVal, '='); pEq != NULL; pEq = strchr(pEq + 1, '=')){ - if(!das_isUrlOrFile(pEq + 1)) continue; - if(pEq == sVal){ /* "=URL", no name */ - fprintf(stderr, PROG ": %s needs URI=URL, a node is named by where it sits " - "in the catalog\n", bMount ? "-m" : "-r"); + const char* sOpt = bMount ? "-m" : "-r"; + const char* pEq = strrchr(sVal, '='); + const char* sName = (pEq != NULL) ? pEq + 1 : NULL; + bool bNamed = false; + if((sName != NULL) && (sName[0] != '\0') && !das_isUrlOrFile(sName)){ + if(das_isUri(sName)) bNamed = true; + else bNamed = (memchr(sVal, '?', (size_t)(pEq - sVal)) == NULL); + } + + if(bNamed){ + char sUrl[URL_SZ]; + size_t uLen = (size_t)(pEq - sVal); + if(uLen >= sizeof(sUrl)) uLen = sizeof(sUrl) - 1; + memcpy(sUrl, sVal, uLen); + sUrl[uLen] = '\0'; + if(!das_isUrlOrFile(sUrl)){ + fprintf(stderr, PROG ": %s takes URL=URI, '%s' before the '=' is not a URL or a file\n", sOpt, sUrl); return false; } - char sUri[URI_SZ]; - size_t uLen = (size_t)(pEq - sVal); - if(uLen >= sizeof(sUri)) uLen = sizeof(sUri) - 1; - memcpy(sUri, sVal, uLen); - sUri[uLen] = '\0'; - if(bMount) return DasCat_mount(pCat, pEq + 1, sUri) == DAS_OKAY; - return DasCat_setRoot(pCat, pEq + 1, sUri) == DAS_OKAY; + if(bMount) return DasCat_mount(pCat, sUrl, sName) == DAS_OKAY; + return DasCat_setSeed(pCat, sUrl, sName) == DAS_OKAY; } - if(bMount){ - fprintf(stderr, PROG ": -m needs URI=URL, see -h\n"); + + /* A location after the '=' is the old order, URI=URL, or "=URL" */ + if((sName != NULL) && das_isUrlOrFile(sName)){ + fprintf(stderr, PROG ": %s takes URL=URI, the location first and the catalog name " + "it stands for after the '='\n", sOpt); return false; } - if(strchr(sVal, '=') != NULL){ - fprintf(stderr, PROG ": the part after '=' must be a URL or a file\n"); + if(bMount){ + fprintf(stderr, PROG ": -m needs URL=URI, see -h\n"); return false; } - if(das_isUrlOrFile(sVal)){ - fprintf(stderr, PROG ": a root needs its catalog URI, -r URI=%s, see -h\n", sVal); + if(das_isUrlOrFile(sVal)) return DasCat_setSeed(pCat, sVal, NULL) == DAS_OKAY; /* a root */ + if(pEq != NULL){ + fprintf(stderr, PROG ": -r takes URL=URI, and '%s' after the '=' is not a catalog name\n", sName); return false; } return DasCat_setPrefix(pCat, sVal) == DAS_OKAY; @@ -966,9 +986,9 @@ static const DasJdo* childStub(const DasNode* pNode, const char* sName) * Without -a it wants das3 streams and settles for anything, two asks. * An -a list is one ask and a set, so a user who lists two formats gets * every source that emits either and names one on the path if that is - * several. The chosen child is loaded through the node's own tree so - * credentials and the cache follow. */ -static DasSrc* pickSrc(DasNode* pNode, const char* sAccept) + * several. The chosen child is loaded by the session so credentials and + * the cache follow. */ +static DasSrc* pickSrc(DasCat* pCat, DasNode* pNode, const char* sAccept) { if(DasNode_isStreamSrc(pNode)) return new_DasSrc_node(pNode); @@ -1006,7 +1026,7 @@ static DasSrc* pickSrc(DasNode* pNode, const char* sAccept) return NULL; } - DasNode* pSub = DasNode_subNode(pNode, aPick[0], NULL, NULL); + DasNode* pSub = DasCat_subNode(pCat, pNode, aPick[0]); if(pSub == NULL){ fprintf(stderr, PROG ": couldn't load source '%s' under %s\n", aPick[0], DasNode_pathUri(pNode)); return NULL; @@ -1022,23 +1042,22 @@ static void explainUnresolved(DasCat* pCat, const char* sPath) char sFullUri[URI_SZ]; strcpyz(sFullUri, DasCat_lastUri(pCat), sizeof(sFullUri)); - /* The common mistake with a named root is asking for something it does - not cover, say so instead of hunting for a parent. A local: name is - never looked up at all, see D2URI_LOCAL. */ + /* The common mistake with a seed is asking for something it does not + cover, say so instead of hunting for a parent */ if(!DasCat_covers(pCat, sFullUri)){ - if(DasCat_rootUri(pCat) == NULL) - daslog_error_v("Nothing is mounted at '%s', and local: names are not looked up " - "in the federated catalog", sFullUri); - else - daslog_error_v("'%s' is outside the -r root %s, use -m to mount it on the " - "federated catalog instead", sFullUri, DasCat_rootUri(pCat)); + daslog_error_v("'%s' is outside the -r seed %s, use -m to mount it on the " + "federated catalog instead", sFullUri, DasCat_prefix(pCat)); return; } const char* sRest = NULL; DasNode* pNode = DasCat_nearest(pCat, &sRest); if((pNode == NULL)||(sRest == NULL)||(sRest[0] == '\0')){ - daslog_error_v("Couldn't resolve catalog path '%s'", sFullUri); + /* A root seed has no URI, so the path to it is empty; name the file */ + if((sFullUri[0] == '\0') && DasCat_hasSeed(pCat)) + daslog_error_v("Couldn't load the seed %s", DasCat_seedUrl(pCat)); + else + daslog_error_v("Couldn't resolve catalog path '%s'", sFullUri); return; } const char* sParentUri = DasCat_lastUri(pCat); @@ -1179,7 +1198,7 @@ int main(int argc, char** argv) const char* sCmd = NULL; const char* sPath = NULL; const char* sCacheArg = NULL; - const char* sRootArg = NULL; + const char* sStartArg = NULL; const char* aMountArg[MAX_MOUNTS]; size_t uMountArg = 0; const char* sAccept = NULL; /* -a, else the two step default in pickSrc */ @@ -1203,7 +1222,7 @@ int main(int argc, char** argv) if(strcmp(sArg, "-q") == 0){ daslog_setlevel(DASLOG_ERROR); bQuiet = true; continue; } if(strcmp(sArg, "-v") == 0){ daslog_setlevel(DASLOG_INFO); g_bVerbose = true; g_uShow = SHOW_ALL; continue; } if(strcmp(sArg, "-f") == 0){ bFresh = true; continue; } - if(strcmp(sArg, "--root") == 0) sArg = "-r"; + if(strcmp(sArg, "--start") == 0) sArg = "-r"; if(strcmp(sArg, "--mount") == 0) sArg = "-m"; if((strcmp(sArg, "-r") == 0)||(strcmp(sArg, "-m") == 0)||(strcmp(sArg, "-s") == 0)|| (strcmp(sArg, "-C") == 0)||(strcmp(sArg, "-p") == 0)||(strcmp(sArg, "-t") == 0)|| @@ -1214,7 +1233,7 @@ int main(int argc, char** argv) } const char* sVal = argv[++i]; switch(sArg[1]){ - case 'r': sRootArg = sVal; break; + case 'r': sStartArg = sVal; break; case 'm': if(uMountArg >= MAX_MOUNTS){ fprintf(stderr, PROG ": at most %d mounts\n", MAX_MOUNTS); @@ -1280,26 +1299,26 @@ int main(int argc, char** argv) pCat = new_DasCat(pMgr, sAgent); /* Environment fallbacks for -r and -m, for people who live in one test - catalog. The root is applied first so a mount can complain about a + catalog. The start is applied first so a mount can complain about a conflicting one. */ - const char* sEnv = getenv(ROOT_ENV); - if((sRootArg == NULL) && (sEnv != NULL) && (sEnv[0] != '\0')) sRootArg = sEnv; + const char* sEnv = getenv(START_ENV); + if((sStartArg == NULL) && (sEnv != NULL) && (sEnv[0] != '\0')) sStartArg = sEnv; sEnv = getenv(MOUNT_ENV); if((uMountArg == 0) && (sEnv != NULL) && (sEnv[0] != '\0')) aMountArg[uMountArg++] = sEnv; - if((sRootArg != NULL) && !applyRootArg(pCat, sRootArg, false)) + if((sStartArg != NULL) && !applyStartArg(pCat, sStartArg, false)) return finish(pCat, RET_USAGE); for(size_t u = 0; u < uMountArg; ++u) - if(!applyRootArg(pCat, aMountArg[u], true)) return finish(pCat, RET_USAGE); + if(!applyStartArg(pCat, aMountArg[u], true)) return finish(pCat, RET_USAGE); if(sCmd == NULL){ fprintf(stderr, PROG ": no command given, use -h for help\n"); return finish(pCat, RET_USAGE); } - if(strcmp(sCmd, "roots") == 0){ - size_t uRoots = 0; - const char** psRoots = das_root_urls(&uRoots); - for(size_t u = 0; u < uRoots; ++u) printf("%s\n", psRoots[u]); + if(strcmp(sCmd, "seeds") == 0){ + size_t uSeeds = 0; + const char** psSeeds = das_seed_urls(&uSeeds); + for(size_t u = 0; u < uSeeds; ++u) printf("%s\n", psSeeds[u]); return finish(pCat, RET_OK); } @@ -1311,7 +1330,8 @@ int main(int argc, char** argv) fprintf(stderr, PROG ": unknown command '%s', use -h for help\n", sCmd); return finish(pCat, RET_USAGE); } - if((bJson || bUrl || bData) && (sPath == NULL) && !DasCat_hasRoot(pCat) && (DasCat_mountCount(pCat) == 0)){ + /* Mounts are consulted on the way, they are not a place to start */ + if((bJson || bUrl || bData) && (sPath == NULL) && !DasCat_hasSeed(pCat)){ fprintf(stderr, PROG ": the %s command needs a PATH, use -h for help\n", sCmd); return finish(pCat, RET_USAGE); } @@ -1378,7 +1398,7 @@ int main(int argc, char** argv) if(!bSrcSet || g_bVerbose) listCatalog(pNode, stdout, sShow); if(bSrcSet){ - DasSrc* pSrc = pickSrc(pNode, sAccept); + DasSrc* pSrc = pickSrc(pCat, pNode, sAccept); if(pSrc == NULL){ /* pickSrc said why; the listing shows what to name */ if(!g_bVerbose) listCatalog(pNode, stdout, sShow); @@ -1429,7 +1449,7 @@ int main(int argc, char** argv) /* url or data. The source outlives nothing here, but it does not need the tree either, so the order of teardown does not matter. */ - DasSrc* pSrc = pickSrc(pNode, sAccept); + DasSrc* pSrc = pickSrc(pCat, pNode, sAccept); if(pSrc == NULL) return finish(pCat, RET_NODE); /* Example selectors among the keys: examples.N, or bare N as the shortcut, From efa52000117f5cc6adf201ab6341c26f126de8ab Mon Sep 17 00:00:00 2001 From: cpiker Date: Wed, 28 Oct 2026 04:14:40 -0500 Subject: [PATCH 13/18] Add test program ./tmp dir creation --- test/TestSource.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/TestSource.c b/test/TestSource.c index f66375a..90328da 100644 --- a/test/TestSource.c +++ b/test/TestSource.c @@ -727,7 +727,9 @@ static void test_req(void) CHECK(new_DasReq_file("test/das_fetch/no_such_file.json") == NULL, "missing file"); daslog_setlevel(DASLOG_WARN); - /* file round trip and a copy */ + /* file round trip and a copy. ./tmp is git ignored, so a fresh + checkout, CI for one, does not have it */ + das_mkdirsto("tmp/TestSource_req.json"); CHECK(DasReq_write(pReq, "tmp/TestSource_req.json") == DAS_OKAY, "write"); pBack = new_DasReq_file("tmp/TestSource_req.json"); CHECK(pBack && streq(DasReq_value(pBack, "time"), "2026-01-03"), "read back from file"); From 8d47efbe20feefdf67da62ab26840cd7f1e85448 Mon Sep 17 00:00:00 2001 From: cpiker Date: Wed, 28 Oct 2026 04:23:49 -0500 Subject: [PATCH 14/18] Remove http server from basic test set --- buildfiles/Darwin.mak | 15 +++++++++++++-- buildfiles/Linux.mak | 15 +++++++++++++-- buildfiles/cmake/das2CTests.cmake | 5 +++-- test/das_cache_test.sh | 18 +++++++++++++++--- 4 files changed, 44 insertions(+), 9 deletions(-) diff --git a/buildfiles/Darwin.mak b/buildfiles/Darwin.mak index 92ac5fd..5178be9 100644 --- a/buildfiles/Darwin.mak +++ b/buildfiles/Darwin.mak @@ -217,7 +217,7 @@ $(DESTDIR)$(INST_NAT_BIN)/%:$(BD)/% # archive. Listing those too would let a missing one, say the object for a # newly added source, go unbuilt while the archive looks up to date. .SECONDARY: $(UTIL_OBJS) -.PHONY: test build +.PHONY: test test_more build ## Explicit Rules ########################################################### @@ -406,7 +406,6 @@ test_main: $(BD) $(BD)/$(TARG).a $(BUILD_TEST_PROGS) $(BULID_UTIL_PROGS) test/das3_info_test.sh $(BD) test/das3_from_das2_test.sh $(BD) test/das_fetch_test.sh $(BD) # test/das_fetch/*.json nodes -> query URLs, offline - test/das_cache_test.sh $(BD) # node cache revalidation against a local python server @echo "INFO: Running unit test for CSVs with variable length item data, $(BD)/das3_csv" $(BD)/das3_csv < examples/ex21_tracers_cdpu_status.d3b > $(BD)/ex21_tracers_cdpu_status.csv @echo "INFO: ===============================================" @@ -414,6 +413,18 @@ test_main: $(BD) $(BD)/$(TARG).a $(BUILD_TEST_PROGS) $(BULID_UTIL_PROGS) @echo "INFO: ===============================================" # Stretch fixtures, see Linux.mak. This target never fails the build on its own. +# Tests that need more than a C toolchain and bash. Not in 'make test', so +# the base suite runs anywhere the library builds; run these by hand or in +# a job that has the extras. +# +# das_cache_test.sh node cache revalidation against a local HTTP server +# written in python3: a second run makes no request, +# an aged copy is renewed by a 304, a changed node is +# fetched again, an unreachable server leaves the old +# copy in use +test_more: $(BD)/das_fetch + test/das_cache_test.sh $(BD) + .PHONY: future future: $(BD) $(BD)/TestV3Read @echo "INFO: Stretch fixtures we don't expect to read yet (test/streams/notimp_*):" diff --git a/buildfiles/Linux.mak b/buildfiles/Linux.mak index 6e443df..d3d08f9 100644 --- a/buildfiles/Linux.mak +++ b/buildfiles/Linux.mak @@ -181,7 +181,7 @@ $(DESTDIR)$(INST_NAT_BIN)/%:$(BD)/% # archive. Listing those too would let a missing one, say the object for a # newly added source, go unbuilt while the archive looks up to date. .SECONDARY: $(UTIL_OBJS) -.PHONY: test build +.PHONY: test test_more build ## Explicit Rules ########################################################### @@ -406,7 +406,6 @@ test_main: $(BD) $(BD)/$(TARG).a $(BUILD_TEST_PROGS) $(BUILD_UTIL_PROGS) test/das3_info_test.sh $(BD) # examples/ex*.info golds for the dataset printers test/das3_from_das2_test.sh $(BD) # examples/ex11 pass-through, metafile gold, reject test/das_fetch_test.sh $(BD) # test/das_fetch/*.json nodes -> query URLs, offline - test/das_cache_test.sh $(BD) # node cache revalidation against a local python server @echo "INFO: Running unit test for CSVs with variable length item data, $(BD)/das3_csv" $(BD)/das3_csv < examples/ex21_tracers_cdpu_status.d3b > $(BD)/ex21_tracers_cdpu_status.csv @echo "INFO: ===============================================" @@ -420,6 +419,18 @@ test_main: $(BD) $(BD)/$(TARG).a $(BUILD_TEST_PROGS) $(BUILD_UTIL_PROGS) # expect them to fail today. Kept out of 'test' (which must stay green) and parked here # so you can watch for one starting to pass -- that's the cue to promote it out of the # notimp_ prefix into the regular suite. This target never fails the build on its own. +# Tests that need more than a C toolchain and bash. Not in 'make test', so +# the base suite runs anywhere the library builds; run these by hand or in +# a job that has the extras. +# +# das_cache_test.sh node cache revalidation against a local HTTP server +# written in python3: a second run makes no request, +# an aged copy is renewed by a 304, a changed node is +# fetched again, an unreachable server leaves the old +# copy in use +test_more: $(BD)/das_fetch + test/das_cache_test.sh $(BD) + .PHONY: future future: $(BD) $(BD)/TestV3Read @echo "INFO: Stretch fixtures we don't expect to read yet (test/streams/notimp_*):" diff --git a/buildfiles/cmake/das2CTests.cmake b/buildfiles/cmake/das2CTests.cmake index 2f805f2..e02ba14 100644 --- a/buildfiles/cmake/das2CTests.cmake +++ b/buildfiles/cmake/das2CTests.cmake @@ -88,8 +88,9 @@ _das_redir(das3_csv_varlen main ) # das_fetch: a saved catalog node and its example 0 become a query URL, no -# network. The full offline suite is test/das_fetch_test.sh (bash only), and -# test/das_cache_test.sh covers node cache revalidation (bash and python3). +# network. The full offline suite is test/das_fetch_test.sh (bash only); +# test/das_cache_test.sh covers node cache revalidation (bash and python3) +# and is outside the base suite, see 'make test_more' in the make files. _das_test(das_fetch_url main das_fetch -C none url test/das_fetch/geomag_flex_0.7.json 0) set_tests_properties(das_fetch_url PROPERTIES PASS_REGULAR_EXPRESSION "read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" diff --git a/test/das_cache_test.sh b/test/das_cache_test.sh index 38ed798..c9084a6 100755 --- a/test/das_cache_test.sh +++ b/test/das_cache_test.sh @@ -40,10 +40,22 @@ CACHE=$DAS_USERDIR/cache/nodes start_server() { # start_server [--no-validators] rm -f "$TMPD/port" - python3 test/das_cache_srv.py --dir "$SRV" --log "$LOG" --port-file "$TMPD/port" "$@" & + python3 test/das_cache_srv.py --dir "$SRV" --log "$LOG" --port-file "$TMPD/port" "$@" \ + 2> "$TMPD/srv.err" & SRVPID=$! - for i in 1 2 3 4 5 6 7 8 9 10; do [ -s "$TMPD/port" ] && break; sleep 0.5; done - [ -s "$TMPD/port" ] || { echo " FAIL: test server did not start"; exit 1; } + # A cold python3 on a CI runner, macOS in particular, can take well over + # five seconds to get to the bind, so the wait is generous; a server + # that dies instead says why in srv.err + n=0 + while [ $n -lt 120 ] && [ ! -s "$TMPD/port" ]; do + kill -0 "$SRVPID" 2>/dev/null || break + sleep 0.5; n=$((n + 1)) + done + if [ ! -s "$TMPD/port" ]; then + echo " FAIL: test server did not start" + [ -s "$TMPD/srv.err" ] && sed 's/^/ srv: /' "$TMPD/srv.err" + exit 1 + fi PORT=$(cat "$TMPD/port") } stop_server() { kill "$SRVPID" 2>/dev/null; wait "$SRVPID" 2>/dev/null; SRVPID=""; } From dbe4dee74ce91bb660546a84b188846aa9bcd440 Mon Sep 17 00:00:00 2001 From: cpiker Date: Wed, 28 Oct 2026 04:43:51 -0500 Subject: [PATCH 15/18] utility function name style guide applied --- das3/catalog.c | 48 ++--- das3/node.c | 128 +++++------ das3/node_priv.h | 8 +- das3/req.c | 116 +++++----- das3/source.c | 550 +++++++++++++++++++++++------------------------ 5 files changed, 425 insertions(+), 425 deletions(-) diff --git a/das3/catalog.c b/das3/catalog.c index 87fa1c9..eb36d73 100644 --- a/das3/catalog.c +++ b/das3/catalog.c @@ -93,7 +93,7 @@ struct das_cat { }; /* Defined further down, used throughout */ -static char* _strdupz(const char* s); +static char* _dascat_strdupz(const char* s); static DasNode* _dascat_loadUrl( const char* sUrl, const char* sPathUri, DasCredMngr* pMgr, const char* sAgent, const DasNodeCache* pCache @@ -430,13 +430,13 @@ static void _DasCat_setNear(DasCat* pThis, const DasNodeWalk* pWalk, DasNode* pF else{ pThis->sNear = das_strdup(sNear); } - pThis->sNearRest = _strdupz(sRest); + pThis->sNearRest = _dascat_strdupz(sRest); } /* A path below a node, with this session's settings. Children get a * slower timeout than roots since there is no second copy of the tree to * fall back to. */ -static DasNode* _dascat_walkFrom(DasCat* pThis, DasNode* pFrom, const char* sRel) +static DasNode* _DasCat_walkFrom(DasCat* pThis, DasNode* pFrom, const char* sRel) { DasNodeWalk* pWalk = new_DasNodeWalk(pFrom, sRel); if(pWalk == NULL) return NULL; @@ -507,7 +507,7 @@ static DasNode* _dascat_loadUrl( if(f.pBytes != NULL){ size_t uLen = 0; const char* p = (const char*)DasAry_getBytesIn(f.pBytes, DIM0, &uLen); - pNode = _DasNode_fromJson(p, uLen, sUrl, f.sFinal, sUri); + pNode = _dasnode_fromJson(p, uLen, sUrl, f.sFinal, sUri); if(pNode != NULL){ pNode->bOwner = true; _dascat_keep(&f, p, uLen); @@ -520,7 +520,7 @@ static DasNode* _dascat_loadUrl( /* ************************************************************************* */ /* Small helpers */ -static void _setStr(char** ps, const char* s) +static void _dascat_setStr(char** ps, const char* s) { free(*ps); *ps = (s != NULL) ? das_strdup(s) : NULL; @@ -528,7 +528,7 @@ static void _setStr(char** ps, const char* s) /* Shorthands are expanded by node.c as well; doing it here too keeps * DasCat_lastUri() the full form even when a resolve fails early. */ -static char* _expand(const char* sIn) +static char* _dascat_expandUri(const char* sIn) { char sBuf[600]; return das_strdup(_das_uri_expand(sIn, sBuf, sizeof(sBuf))); @@ -648,7 +648,7 @@ int DasCat_cacheAge(const DasCat* pThis) /* Roots are built on first use, so the policy is frozen by then */ /* das_strdup() gives NULL for "", and these fields promise a string */ -static char* _strdupz(const char* s) +static char* _dascat_strdupz(const char* s) { return ((s == NULL)||(s[0] == '\0')) ? (char*)calloc(1, 1) : das_strdup(s); } @@ -676,7 +676,7 @@ DasErrCode DasCat_setPrefix(DasCat* pThis, const char* sUri) return DASERR_CAT; } free(pThis->sPrefix); - pThis->sPrefix = _expand(sUri); + pThis->sPrefix = _dascat_expandUri(sUri); /* A prefix is joined to relative paths by plain concatenation, so it must end in the container's separator. A bare shorthand names the container; its separator is known, supply it. */ @@ -721,13 +721,13 @@ static DasErrCode _DasCat_addMount(DasCat* pThis, const char* sUrl, const char* if(a == NULL) return das_error(DASERR_CAT, "Couldn't grow the mount list"); pThis->aMounts = a; a[pThis->uMounts].sUrl = das_strdup(sUrl); - a[pThis->uMounts].sUri = _expand(sUri); + a[pThis->uMounts].sUri = _dascat_expandUri(sUri); pThis->uMounts += 1; return DAS_OKAY; } /* The checks shared by seeds and mounts */ -static bool _DasCat_locationOk(const char* sUrl, const char* sUri, bool bUriNeeded) +static bool _dascat_locationOk(const char* sUrl, const char* sUri, bool bUriNeeded) { if(!das_isUrlOrFile(sUrl)){ daslog_error_v("A node is loaded from a URL or a file, '%s' is neither", sUrl ? sUrl : ""); @@ -754,9 +754,9 @@ DasErrCode DasCat_setSeed(DasCat* pThis, const char* sUrl, const char* sUri) } if(pThis->bPrefixGiven) daslog_info_v("Relative paths started at %s, now they start at the seed", pThis->sPrefix); - if(!_DasCat_locationOk(sUrl, sUri, false)) return DASERR_CAT; + if(!_dascat_locationOk(sUrl, sUri, false)) return DASERR_CAT; pThis->sSeedUrl = das_strdup(sUrl); - pThis->sSeedUri = ((sUri != NULL) && (sUri[0] != '\0')) ? _expand(sUri) : (char*)calloc(1, 1); + pThis->sSeedUri = ((sUri != NULL) && (sUri[0] != '\0')) ? _dascat_expandUri(sUri) : (char*)calloc(1, 1); pThis->bSeed = true; return DAS_OKAY; } @@ -766,7 +766,7 @@ const char* DasCat_seedUrl(const DasCat* pThis){ return pThis->sSeedUrl; } DasErrCode DasCat_mount(DasCat* pThis, const char* sUrl, const char* sUri) { if(_DasCat_frozen(pThis)) return DASERR_CAT; - if(!_DasCat_locationOk(sUrl, sUri, true)) return DASERR_CAT; + if(!_dascat_locationOk(sUrl, sUri, true)) return DASERR_CAT; return _DasCat_addMount(pThis, sUrl, sUri); } @@ -806,7 +806,7 @@ static DasNode* _DasCat_table(DasCat* pThis) s = aUrl[j]; aUrl[j] = aUrl[j-1]; aUrl[j-1] = s; } } - pThis->pTable = _DasNode_newMountTable(aUrl, aUri, pThis->uMounts); + pThis->pTable = _dasnode_newMountTable(aUrl, aUri, pThis->uMounts); free(aUrl); free(aUri); return _DasCat_keep(pThis, pThis->pTable); @@ -822,7 +822,7 @@ static DasNode* _DasCat_byUri(DasCat* pThis, const char* sUri) if(_DasCat_mountFor(pThis, sUri) >= 0){ DasNode* pTable = _DasCat_table(pThis); if(pTable == NULL) return NULL; - return _dascat_walkFrom(pThis, pTable, sUri); + return _DasCat_walkFrom(pThis, pTable, sUri); } if(pThis->bSeed){ if(!DasCat_covers(pThis, sUri)){ @@ -835,17 +835,17 @@ static DasNode* _DasCat_byUri(DasCat* pThis, const char* sUri) which no walk can start from, so it is handed back directly */ const char* sRest = sUri + strlen(pThis->sSeedUri); if(sRest[0] == '\0') return pSeed; - return _dascat_walkFrom(pThis, pSeed, sRest); + return _DasCat_walkFrom(pThis, pSeed, sRest); } DasNode* pDas2 = _DasCat_das2(pThis); if(pDas2 == NULL) return NULL; - return _dascat_walkFrom(pThis, pDas2, sUri); + return _DasCat_walkFrom(pThis, pDas2, sUri); } DasNode* DasCat_resolve(DasCat* pThis, const char* sPath) { bool bNoPath = (sPath == NULL)||(sPath[0] == '\0'); - _setStr(&pThis->sLastPath, bNoPath ? NULL : sPath); + _dascat_setStr(&pThis->sLastPath, bNoPath ? NULL : sPath); free(pThis->sLastUri); pThis->sLastUri = NULL; free(pThis->sNear); pThis->sNear = NULL; free(pThis->sNearRest); pThis->sNearRest = NULL; @@ -863,16 +863,16 @@ DasNode* DasCat_resolve(DasCat* pThis, const char* sPath) * mounts or nowhere */ if(pThis->bSeed){ if(bNoPath){ - pThis->sLastUri = _strdupz(pThis->sSeedUri); + pThis->sLastUri = _dascat_strdupz(pThis->sSeedUri); return _DasCat_seedNode(pThis); } if(das_isUri(sPath)){ - pThis->sLastUri = _expand(sPath); + pThis->sLastUri = _dascat_expandUri(sPath); return _DasCat_byUri(pThis, pThis->sLastUri); } DasNode* pSeed = _DasCat_seedNode(pThis); if(pSeed == NULL){ - pThis->sLastUri = _strdupz(pThis->sSeedUri); + pThis->sLastUri = _dascat_strdupz(pThis->sSeedUri); return NULL; } /* Names below a root are the path from it; below a prefix the @@ -898,7 +898,7 @@ DasNode* DasCat_resolve(DasCat* pThis, const char* sPath) pThis->sLastUri[--u] = '\0'; } else if(das_isUri(sPath)) - pThis->sLastUri = _expand(sPath); + pThis->sLastUri = _dascat_expandUri(sPath); else pThis->sLastUri = das_string("%s%s", pThis->sPrefix, sPath); @@ -926,7 +926,7 @@ DasNode* DasCat_nearest(DasCat* pThis, const char** psRest) last path and URI say so with an empty string */ DasNode* pNode = pThis->pNearNode; pThis->pNearNode = NULL; - _setStr(&pThis->sLastPath, NULL); + _dascat_setStr(&pThis->sLastPath, NULL); free(pThis->sLastUri); pThis->sLastUri = (char*)calloc(1, 1); if(psRest != NULL) *psRest = pThis->sNearRest; @@ -972,7 +972,7 @@ DasNode* DasCat_subNode(DasCat* pThis, DasNode* pNode, const char* sRelPath) daslog_error("No node to load a child of"); return NULL; } - return _dascat_walkFrom(pThis, pNode, sRelPath); + return _DasCat_walkFrom(pThis, pNode, sRelPath); } /* ************************************************************************* */ diff --git a/das3/node.c b/das3/node.c index 1fa9170..b746fc9 100644 --- a/das3/node.c +++ b/das3/node.c @@ -61,7 +61,7 @@ const char** das_seed_urls(size_t* pLen){ } -static das_node_type_e das_node_type(const char* sType){ +static das_node_type_e _dasnode_type(const char* sType){ if(sType == NULL) return d2node_inv; if(strcmp(sType, D2CV_TYPE_CATALOG) == 0) return d2node_catalog; if(strcmp(sType, D2CV_TYPE_COLLECTION) == 0) return d2node_collection; @@ -142,7 +142,7 @@ const char* DasNode_pathUri(const DasNode* pThis) } /* helper, git a string value from the root of the dom */ -static const char* DasNode_rootStr(const DasNode* pThis, const char* sStr) +static const char* _DasNode_rootStr(const DasNode* pThis, const char* sStr) { if(!DasNode_isJson(pThis)){ daslog_error("Non-JSON nodes not supported at this time"); @@ -160,29 +160,29 @@ static const char* DasNode_rootStr(const DasNode* pThis, const char* sStr) const char* DasNode_name(const DasNode* pThis){ - return DasNode_rootStr(pThis, D2FRAG_NAME); + return _DasNode_rootStr(pThis, D2FRAG_NAME); } const char* DasNode_label(const DasNode* pThis){ - const char* pLabel = DasNode_rootStr(pThis, D2FRAG_LABEL); + const char* pLabel = _DasNode_rootStr(pThis, D2FRAG_LABEL); if(pLabel != NULL) return pLabel; - return DasNode_rootStr(pThis, D2FRAG_NAME); + return _DasNode_rootStr(pThis, D2FRAG_NAME); } const char* DasNode_title(const DasNode* pThis){ - return DasNode_rootStr(pThis, D2FRAG_TITLE); + return _DasNode_rootStr(pThis, D2FRAG_TITLE); } const char* DasNode_summary(const DasNode* pThis){ - return DasNode_rootStr(pThis, D2FRAG_DESC); + return _DasNode_rootStr(pThis, D2FRAG_DESC); } const char* DasNode_srcUrl(const DasNode* pThis){ return pThis->sURL;} const char* DasNode_version(const DasNode* pThis){ - return DasNode_rootStr(pThis, D2FRAG_VERSION); + return _DasNode_rootStr(pThis, D2FRAG_VERSION); } const char* DasNode_pathSep(const DasNode* pThis){ @@ -232,7 +232,7 @@ das_node_type_e DasNode_subType(const DasNode* pThis, size_t uIdx) { const DasJdo* pStub = DasNode_subStub(pThis, uIdx); if(pStub == NULL) return d2node_inv; - return das_node_type( DasJdo_string( DasJdo_get(pStub, D2FRAG_TYPE) ) ); + return _dasnode_type( DasJdo_string( DasJdo_get(pStub, D2FRAG_TYPE) ) ); } /* ************************************************************************* */ @@ -250,12 +250,12 @@ das_node_type_e DasNode_subType(const DasNode* pThis, size_t uIdx) * model. None is written; this only says where. */ /* ---- 2018 Collection shim, begin. When those catalogs are regenerated - delete down to the end marker, plus the _DasNode_upgrade() call in - _DasNode_fromJson() and the pShim field: a table with no entries is + delete down to the end marker, plus the _dasnode_upgrade() call in + _dasnode_fromJson() and the pShim field: a table with no entries is not C99, so the next translator brings the table back. The comment above stays; it says where. ---- */ -static bool _ieq(const char* a, const char* b) +static bool _dasnode_ieq(const char* a, const char* b) { if((a == NULL)||(b == NULL)) return false; for(; *a && *b; ++a, ++b) @@ -278,20 +278,20 @@ struct provides_patch { /* "provides": ["mime"], all in one block */ das_json_str mime; }; -static const char* _impliedProvides(const DasJdo* pEntry) +static const char* _dasnode_impliedProvides(const DasJdo* pEntry) { if((pEntry == NULL)||(pEntry->type != das_json_type_dict)) return NULL; - if(!_ieq(DasJdo_string(DasJdo_get(pEntry, D2FRAG_TYPE)), D2CV_TYPE_STREAM)) return NULL; + if(!_dasnode_ieq(DasJdo_string(DasJdo_get(pEntry, D2FRAG_TYPE)), D2CV_TYPE_STREAM)) return NULL; if(DasJdo_get(pEntry, "provides") != NULL) return NULL; const char* sConv = DasJdo_string(DasJdo_get(pEntry, "convention")); - if(_ieq(sConv, "das2/2.2")) return "application/vnd.das2.das2stream"; - if(_ieq(sConv, "hapi/2.0")) return "application/vnd.hapi.stream"; + if(_dasnode_ieq(sConv, "das2/2.2")) return "application/vnd.das2.das2stream"; + if(_dasnode_ieq(sConv, "hapi/2.0")) return "application/vnd.hapi.stream"; return NULL; } /* TODO: Remove this particular upgrade function once no old style collection catalog nodes exist in the wild */ -static bool _upgradeCollection(DasJdo* pRoot, void** ppShim) +static bool _dasnode_upgradeCollection(DasJdo* pRoot, void** ppShim) { const DasJdo* pSrcs = DasJdo_get(pRoot, D2FRAG_SOURCES); if((pSrcs == NULL)||(pSrcs->type != das_json_type_dict)) return true; /* reported later */ @@ -299,7 +299,7 @@ static bool _upgradeCollection(DasJdo* pRoot, void** ppShim) size_t uNeed = 0; const das_json_dict_el* pEl = NULL; for(pEl = DasJdo_dictFirst(pSrcs); pEl; pEl = pEl->next) - if(_impliedProvides(pEl->value) != NULL) ++uNeed; + if(_dasnode_impliedProvides(pEl->value) != NULL) ++uNeed; if(uNeed == 0) return true; struct provides_patch* aPatch = (struct provides_patch*)calloc( @@ -310,7 +310,7 @@ static bool _upgradeCollection(DasJdo* pRoot, void** ppShim) size_t u = 0; for(pEl = DasJdo_dictFirst(pSrcs); pEl; pEl = pEl->next){ - const char* sMime = _impliedProvides(pEl->value); + const char* sMime = _dasnode_impliedProvides(pEl->value); if(sMime == NULL) continue; struct provides_patch* p = aPatch + u++; p->name.string = "provides"; p->name.string_size = 8; @@ -340,12 +340,12 @@ static const struct { double rThrough; /* versions up to this one */ bool (*fn)(DasJdo* pRoot, void** ppShim); } g_upgrades[] = { - { d2node_collection, 0.5, _upgradeCollection }, + { d2node_collection, 0.5, _dasnode_upgradeCollection }, }; /* False only when memory ran out. An unversioned document counts as the oldest, since versions were added after these layouts. */ -static bool _DasNode_upgrade(DasJdo* pRoot, das_node_type_e nType, void** ppShim) +static bool _dasnode_upgrade(DasJdo* pRoot, das_node_type_e nType, void** ppShim) { const char* sVer = DasJdo_string(DasJdo_get(pRoot, D2FRAG_VERSION)); double rVer = (sVer == NULL) ? 0.0 : das_strtod_c(sVer, NULL); @@ -360,7 +360,7 @@ static bool _DasNode_upgrade(DasJdo* pRoot, das_node_type_e nType, void** ppShim /* Build a node from JSON text. sUrl is where it came from and sFinalUrl * the same after redirects, both for messages. sPathUri may be NULL. The * node is not marked as owned; constructors decide that. */ -DasNode* _DasNode_fromJson( +DasNode* _dasnode_fromJson( const char* pBytes, size_t uBytes, const char* sUrl, const char* sFinalUrl, const char* sPathUri ){ @@ -390,7 +390,7 @@ DasNode* _DasNode_fromJson( return NULL; } - das_node_type_e nType = das_node_type( sType ); + das_node_type_e nType = _dasnode_type( sType ); if( nType == d2node_inv){ daslog_error_v("Error in catalog object from %s, 'TYPE' value %s is " "unknown", sFinalUrl, sType); @@ -401,7 +401,7 @@ DasNode* _DasNode_fromJson( /* Older layouts are rewritten here so the rest of the node code, and everything built on it, reads only the current one. */ void* pShim = NULL; - if(!_DasNode_upgrade(pRoot, nType, &pShim)){ + if(!_dasnode_upgrade(pRoot, nType, &pShim)){ free(pRoot); return NULL; } @@ -579,7 +579,7 @@ DasNode* new_DasNode_json( if((sPathUri != NULL) && (sPathUri[0] != '\0')) _das_uri_expand(sPathUri, sUri, sizeof(sUri)); if(sUrl == NULL) sUrl = ""; - DasNode* pNode = _DasNode_fromJson(pBytes, uBytes, sUrl, sUrl, sUri); + DasNode* pNode = _dasnode_fromJson(pBytes, uBytes, sUrl, sUrl, sUri); if(pNode) pNode->bOwner = true; return pNode; } @@ -608,7 +608,7 @@ void _DasNode_delTree(DasNode* pThis) dec_DasAry(pCat->pSubNodes); dec_DasAry(pCat->pSubPaths); } - /* Every node holds a JSON DOM, see _DasNode_upgrade(); the shim holds + /* Every node holds a JSON DOM, see _dasnode_upgrade(); the shim holds whatever was added to it on load. */ free(pThis->pDom); free(pThis->pShim); @@ -657,7 +657,7 @@ const char* _das_uri_expand(const char* sIn, char* sOut, size_t uOut) /* The mount table */ /* Append a JSON string literal, escaping the two characters that matter */ -static size_t _jsonStr(char* sOut, size_t uOut, const char* sIn) +static size_t _dasnode_jsonStr(char* sOut, size_t uOut, const char* sIn) { size_t u = 0; if(u < uOut - 1) sOut[u++] = '"'; @@ -671,7 +671,7 @@ static size_t _jsonStr(char* sOut, size_t uOut, const char* sIn) } /* read comments in node_priv.h header on this function first before editing it */ -DasNode* _DasNode_newMountTable(const char** aUrl, const char** aUri, size_t uMounts) +DasNode* _dasnode_newMountTable(const char** aUrl, const char** aUri, size_t uMounts) { if(uMounts == 0){ daslog_error("A mount table needs at least one mount"); @@ -724,9 +724,9 @@ DasNode* _DasNode_newMountTable(const char** aUrl, const char** aUri, size_t uMo for(size_t i = 0; i < uMounts; ++i){ size_t k = aOrder[i]; if(i > 0) u += snprintf(sJson + u, uJson - u, ","); - u += _jsonStr(sJson + u, uJson - u, aUri[k]); + u += _dasnode_jsonStr(sJson + u, uJson - u, aUri[k]); u += snprintf(sJson + u, uJson - u, ":{\"type\":\"Catalog\",\"urls\":["); - u += _jsonStr(sJson + u, uJson - u, aUrl[k]); + u += _dasnode_jsonStr(sJson + u, uJson - u, aUrl[k]); u += snprintf(sJson + u, uJson - u, "]}"); } u += snprintf(sJson + u, uJson - u, "}}"); @@ -737,7 +737,7 @@ DasNode* _DasNode_newMountTable(const char** aUrl, const char** aUri, size_t uMo } /* Nothing is fetched here, the first walk through the table does that */ - DasNode* pTable = _DasNode_fromJson(sJson, u, "memory:mounts", "memory:mounts", ""); + DasNode* pTable = _dasnode_fromJson(sJson, u, "memory:mounts", "memory:mounts", ""); free(sJson); if(pTable != NULL) pTable->bOwner = true; return pTable; @@ -833,7 +833,7 @@ static bool _DasCatNode_hasLoaded(const DasCatNode* pCat, const char* sName) } /* Candidate number n of a frame, counting string entries only */ -static const char* _walk_url(const DasNodeWalk* pThis, const struct walk_frame* pF, size_t n) +static const char* _DasNodeWalk_url(const DasNodeWalk* pThis, const struct walk_frame* pF, size_t n) { if(pF->pUrls == NULL) return (n < pThis->uSeedUrl) ? pThis->aSeedUrl[n] : NULL; size_t k = 0; @@ -845,7 +845,7 @@ static const char* _walk_url(const DasNodeWalk* pThis, const struct walk_frame* return NULL; } -static size_t _walk_countUrls(const DasNode* pParent, const char* sChild, const DasJdo* pUrls) +static size_t _DasNode_countUrls(const DasNode* pParent, const char* sChild, const DasJdo* pUrls) { size_t u = 0; int i = 0; @@ -863,7 +863,7 @@ static size_t _walk_countUrls(const DasNode* pParent, const char* sChild, const * such as "a" for "ab/x" is not on the path. Whether anything under it * matches does not matter: "uiowa" is the nearest node for * "uiowa/nope/deeper" even though it lists no "nope". */ -static void _walk_nearSet(DasNodeWalk* pThis, size_t uDepth, const DasNode* pNode, const char* sRest) +static void _DasNodeWalk_nearSet(DasNodeWalk* pThis, size_t uDepth, const DasNode* pNode, const char* sRest) { if(uDepth <= pThis->uDeepest) return; pThis->uDeepest = uDepth; @@ -873,7 +873,7 @@ static void _walk_nearSet(DasNodeWalk* pThis, size_t uDepth, const DasNode* pNod /* For a child met on the way: it is on the path only if the rest continues * below it with its separator */ -static void _walk_near(DasNodeWalk* pThis, size_t uDepth, const DasNode* pNode, const char* sRem) +static void _DasNodeWalk_near(DasNodeWalk* pThis, size_t uDepth, const DasNode* pNode, const char* sRem) { const char* sSep = ((const DasCatNode*)pNode)->sPathSep; size_t uSep = strlen(sSep); @@ -881,10 +881,10 @@ static void _walk_near(DasNodeWalk* pThis, size_t uDepth, const DasNode* pNode, if(strncmp(sRem, sSep, uSep) != 0) return; sRem += uSep; } - _walk_nearSet(pThis, uDepth, pNode, sRem); + _DasNodeWalk_nearSet(pThis, uDepth, pNode, sRem); } -static bool _walk_push(DasNodeWalk* pThis, DasNode* pParent, const char* sRest) +static bool _DasNodeWalk_push(DasNodeWalk* pThis, DasNode* pParent, const char* sRest) { if(pThis->uFrames == _WALK_DEPTH){ daslog_error_v("Catalog path %s is more than %d levels deep", pThis->sPath, _WALK_DEPTH); @@ -901,14 +901,14 @@ static bool _walk_push(DasNodeWalk* pThis, DasNode* pParent, const char* sRest) if((uSep > 0) && (strncmp(sRest, sSep, uSep) == 0)) sRest += uSep; pF->sRest = sRest; pThis->uFrames += 1; - _walk_nearSet(pThis, pThis->uFrames, pParent, sRest); /* already stripped */ + _DasNodeWalk_nearSet(pThis, pThis->uFrames, pParent, sRest); /* already stripped */ return true; } /* Back out one level. The frame above drops the node it descended into and moves to its next candidate; a frame that descended into an already loaded child has nothing to drop. */ -static void _walk_pop(DasNodeWalk* pThis) +static void _DasNodeWalk_pop(DasNodeWalk* pThis) { struct walk_frame* pF = pThis->aFrame + (pThis->uFrames - 1); if(pF->pNode != NULL){ _DasNode_delTree(pF->pNode); pF->pNode = NULL; } @@ -922,7 +922,7 @@ static void _walk_pop(DasNodeWalk* pThis) } } -static void _walk_finish(DasNodeWalk* pThis, DasNode* pResult) +static void _DasNodeWalk_finish(DasNodeWalk* pThis, DasNode* pResult) { for(size_t k = 0; k < pThis->uFrames; ++k){ struct walk_frame* pF = pThis->aFrame + k; @@ -936,7 +936,7 @@ static void _walk_finish(DasNodeWalk* pThis, DasNode* pResult) } /* Move to the next state: a fetch step, done, or failed */ -static void _walk_advance(DasNodeWalk* pThis) +static void _DasNodeWalk_advance(DasNodeWalk* pThis) { while(pThis->uFrames > 0){ struct walk_frame* pF = pThis->aFrame + (pThis->uFrames - 1); @@ -946,7 +946,7 @@ static void _walk_advance(DasNodeWalk* pThis) pThis->state = das_walk_fetch; return; } - if(pF->pParent == NULL){ _walk_pop(pThis); continue; } /* every seed URL tried */ + if(pF->pParent == NULL){ _DasNodeWalk_pop(pThis); continue; } /* every seed URL tried */ DasCatNode* pCat = (DasCatNode*)pF->pParent; @@ -965,10 +965,10 @@ static void _walk_advance(DasNodeWalk* pThis) if(strncmp(sName, pF->sRest, uLen) != 0) continue; DasNode* pKid = *((DasNode**)DasAry_getAt(pCat->pSubNodes, vtUnknown, IDX0(i))); const char* sRem = pF->sRest + uLen; - if(sRem[0] == '\0'){ _walk_finish(pThis, pKid); return; } - if(DasNode_isCatalog(pKid)) _walk_near(pThis, pThis->uFrames + 1, pKid, sRem); + if(sRem[0] == '\0'){ _DasNodeWalk_finish(pThis, pKid); return; } + if(DasNode_isCatalog(pKid)) _DasNodeWalk_near(pThis, pThis->uFrames + 1, pKid, sRem); if(!_DasNode_subMatch(pKid, sRem)) continue; - if(_walk_push(pThis, pKid, sRem)){ bPushed = true; break; } + if(_DasNodeWalk_push(pThis, pKid, sRem)){ bPushed = true; break; } } if(bPushed) continue; pF->bListing = true; @@ -1008,7 +1008,7 @@ static void _walk_advance(DasNodeWalk* pThis) pF->sChild = sChild; pF->pUrls = pUrls; pF->iUrl = 0; - pF->uUrls = _walk_countUrls(pF->pParent, sChild, pUrls); + pF->uUrls = _DasNode_countUrls(pF->pParent, sChild, pUrls); /* A root has no URI, so its children are named by bare name and the levels below by their path from the root */ if(pF->pParent->sPath[0] == '\0') @@ -1028,12 +1028,12 @@ static void _walk_advance(DasNodeWalk* pThis) const char* sLabel = DasNode_label(pF->pParent); daslog_info_v("Node %s (URI '%s') has no child node that starts with %s", sLabel ? sLabel : "", pF->pParent->sPath, pF->sRest); - _walk_pop(pThis); + _DasNodeWalk_pop(pThis); } pThis->state = das_walk_failed; } -static DasNodeWalk* _walk_alloc(DasNode* pFrom, const char* sRelPath) +static DasNodeWalk* _dasnodewalk_alloc(DasNode* pFrom, const char* sRelPath) { DasNodeWalk* pThis = (DasNodeWalk*)calloc(1, sizeof(DasNodeWalk)); if(pThis == NULL){ @@ -1066,7 +1066,7 @@ DasNodeWalk* new_DasNodeWalk(DasNode* pFrom, const char* sRelPath) DasNode_label(pFrom), DasNode_srcUrl(pFrom)); return NULL; } - DasNodeWalk* pThis = _walk_alloc(pFrom, sRelPath); + DasNodeWalk* pThis = _dasnodewalk_alloc(pFrom, sRelPath); if(pThis == NULL) return NULL; if(pThis->sPath[0] == '\0'){ @@ -1074,14 +1074,14 @@ DasNodeWalk* new_DasNodeWalk(DasNode* pFrom, const char* sRelPath) pThis->state = das_walk_done; return pThis; } - _walk_push(pThis, pFrom, pThis->sPath); /* cannot be too deep at 0 */ - _walk_advance(pThis); + _DasNodeWalk_push(pThis, pFrom, pThis->sPath); /* cannot be too deep at 0 */ + _DasNodeWalk_advance(pThis); return pThis; } DasNodeWalk* new_DasNodeWalk_seed(void) { - DasNodeWalk* pThis = _walk_alloc(NULL, NULL); + DasNodeWalk* pThis = _dasnodewalk_alloc(NULL, NULL); if(pThis == NULL) return NULL; pThis->aSeedUrl = das_seed_urls(&pThis->uSeedUrl); @@ -1092,7 +1092,7 @@ DasNodeWalk* new_DasNodeWalk_seed(void) pF->bListing = true; pF->uUrls = pThis->uSeedUrl; pThis->uFrames = 1; - _walk_advance(pThis); + _DasNodeWalk_advance(pThis); return pThis; } @@ -1121,7 +1121,7 @@ const char* DasNodeWalk_url(const DasNodeWalk* pThis, size_t uIdx) { if(uIdx >= DasNodeWalk_urlCount(pThis)) return NULL; const struct walk_frame* pF = pThis->aFrame + (pThis->uFrames - 1); - return _walk_url(pThis, pF, pF->iUrl + uIdx); + return _DasNodeWalk_url(pThis, pF, pF->iUrl + uIdx); } const char* DasNodeWalk_uri(const DasNodeWalk* pThis) @@ -1147,10 +1147,10 @@ DasErrCode DasNodeWalk_feed( } struct walk_frame* pF = pThis->aFrame + (pThis->uFrames - 1); - const char* sUrl = _walk_url(pThis, pF, pF->iUrl + uIdx); + const char* sUrl = _DasNodeWalk_url(pThis, pF, pF->iUrl + uIdx); if(sFinalUrl == NULL) sFinalUrl = sUrl; - DasNode* pNode = _DasNode_fromJson(pBytes, uBytes, sUrl, sFinalUrl, pF->sUri); + DasNode* pNode = _dasnode_fromJson(pBytes, uBytes, sUrl, sFinalUrl, pF->sUri); if((pNode != NULL) && (pF->pParent == NULL) && (pNode->nType != d2node_catalog)){ daslog_info_v("%s is not a Catalog node, skipping it as a seed", sUrl); _DasNode_delTree(pNode); @@ -1158,18 +1158,18 @@ DasErrCode DasNodeWalk_feed( } if(pNode == NULL){ pF->iUrl += uIdx + 1; - _walk_advance(pThis); + _DasNodeWalk_advance(pThis); return DASERR_NODE; } pF->iUrl += uIdx; /* the one in hand; a back out moves past it */ pF->pNode = pNode; const char* sRem = (pF->pParent == NULL) ? pF->sRest : pF->sRest + strlen(pF->sChild); - if(sRem[0] == '\0'){ _walk_finish(pThis, pNode); return DAS_OKAY; } + if(sRem[0] == '\0'){ _DasNodeWalk_finish(pThis, pNode); return DAS_OKAY; } - if(DasNode_isCatalog(pNode)) _walk_near(pThis, pThis->uFrames + 1, pNode, sRem); - if(DasNode_isCatalog(pNode) && _DasNode_subMatch(pNode, sRem) && _walk_push(pThis, pNode, sRem)){ - _walk_advance(pThis); + if(DasNode_isCatalog(pNode)) _DasNodeWalk_near(pThis, pThis->uFrames + 1, pNode, sRem); + if(DasNode_isCatalog(pNode) && _DasNode_subMatch(pNode, sRem) && _DasNodeWalk_push(pThis, pNode, sRem)){ + _DasNodeWalk_advance(pThis); return DAS_OKAY; } @@ -1179,7 +1179,7 @@ DasErrCode DasNodeWalk_feed( _DasNode_delTree(pNode); pF->pNode = NULL; pF->iUrl += 1; - _walk_advance(pThis); + _DasNodeWalk_advance(pThis); return DAS_OKAY; } @@ -1187,9 +1187,9 @@ void DasNodeWalk_fail(DasNodeWalk* pThis, size_t uIdx, const char* sWhy) { if(uIdx >= DasNodeWalk_urlCount(pThis)) return; struct walk_frame* pF = pThis->aFrame + (pThis->uFrames - 1); - daslog_debug_v("%s: %s", _walk_url(pThis, pF, pF->iUrl + uIdx), sWhy ? sWhy : "not available"); + daslog_debug_v("%s: %s", _DasNodeWalk_url(pThis, pF, pF->iUrl + uIdx), sWhy ? sWhy : "not available"); pF->iUrl += uIdx + 1; - _walk_advance(pThis); + _DasNodeWalk_advance(pThis); } DasNode* DasNodeWalk_result(const DasNodeWalk* pThis) diff --git a/das3/node_priv.h b/das3/node_priv.h index ae97443..9333041 100644 --- a/das3/node_priv.h +++ b/das3/node_priv.h @@ -39,7 +39,7 @@ * The type is checked before casting, see DasNode_isCatalog(). Terminal * nodes (stream sources, file aggregations) are a bare struct das_node. * - * Loading. Every node begins as a JSON document that _DasNode_fromJson() + * Loading. Every node begins as a JSON document that _dasnode_fromJson() * turns into a struct of the right type. Where the bytes come from is * not node.c's business: catalog.c reads files, the cache and HTTP, a * browser build hands them to new_DasNode_json(). The parsed DasJdo is @@ -72,7 +72,7 @@ struct das_node { it, freed only by del_DasNode() */ void* pDom; /* the parsed document, a DasJdo, current layout */ void* pShim; /* elements added on load that the parser did not - allocate, see _DasNode_upgrade(). May be NULL */ + allocate, see _dasnode_upgrade(). May be NULL */ }; /* Catalog nodes have extra arrays to cache nodes */ @@ -101,7 +101,7 @@ typedef struct das_catalog { * form. sUrl is where the text came from and sFinalUrl the same after * redirects, both for messages. sPathUri may be NULL. The node is not * marked as owned, the caller decides that. */ -DasNode* _DasNode_fromJson( +DasNode* _dasnode_fromJson( const char* pBytes, size_t uBytes, const char* sUrl, const char* sFinalUrl, const char* sPathUri ); @@ -181,6 +181,6 @@ const char* _das_uri_expand(const char* sIn, char* sOut, size_t uOut); * or the document could not be allocated. Nothing is fetched at * this point; the mounted nodes load on first lookup. */ -DasNode* _DasNode_newMountTable(const char** aUrl, const char** aUri, size_t uMounts); +DasNode* _dasnode_newMountTable(const char** aUrl, const char** aUri, size_t uMounts); #endif /* _das_node_priv_h_ */ diff --git a/das3/req.c b/das3/req.c index 312eb51..3cb3306 100644 --- a/das3/req.c +++ b/das3/req.c @@ -32,7 +32,7 @@ /* ************************************************************************* */ /* Helpers */ -static bool _strieq(const char* a, const char* b) +static bool _dasreq_strieq(const char* a, const char* b) { if((a == NULL)||(b == NULL)) return false; while(*a && *b){ @@ -45,15 +45,15 @@ static bool _strieq(const char* a, const char* b) /* das_strdup() returns NULL for NULL and for "", and here "" is a value: an empty stamp, an unfilled entry. Every string in a requisition goes through this so accessors can hand them straight back. */ -static char* _dupz(const char* s) +static char* _dasreq_dupz(const char* s) { if((s == NULL)||(s[0] == '\0')) return (char*)calloc(1, 1); return das_strdup(s); } -static bool _setStr(char** ps, const char* s) +static bool _dasreq_setStr(char** ps, const char* s) { - char* sNew = _dupz(s); + char* sNew = _dasreq_dupz(s); if(sNew == NULL) return false; free(*ps); *ps = sNew; @@ -90,10 +90,10 @@ DasReq* new_DasReq(void) { DasReq* pThis = (DasReq*)calloc(1, sizeof(DasReq)); if(pThis == NULL) return NULL; - pThis->sPath = _dupz(NULL); - pThis->sUrl = _dupz(NULL); - pThis->sVersion = _dupz(NULL); - pThis->sLabel = _dupz(NULL); + pThis->sPath = _dasreq_dupz(NULL); + pThis->sUrl = _dasreq_dupz(NULL); + pThis->sVersion = _dasreq_dupz(NULL); + pThis->sLabel = _dasreq_dupz(NULL); if(!pThis->sPath || !pThis->sUrl || !pThis->sVersion || !pThis->sLabel){ del_DasReq(pThis); return NULL; @@ -139,21 +139,21 @@ DasReq* copy_DasReq(const DasReq* pThis) void DasReq_setSrc(DasReq* pThis, const char* sPath, const char* sUrl, const char* sVersion) { - _setStr(&pThis->sPath, sPath); - _setStr(&pThis->sUrl, sUrl); - _setStr(&pThis->sVersion, sVersion); + _dasreq_setStr(&pThis->sPath, sPath); + _dasreq_setStr(&pThis->sUrl, sUrl); + _dasreq_setStr(&pThis->sVersion, sVersion); } const char* DasReq_srcPath(const DasReq* pThis){ return pThis->sPath; } const char* DasReq_srcUrl(const DasReq* pThis){ return pThis->sUrl; } const char* DasReq_srcVersion(const DasReq* pThis){ return pThis->sVersion; } const char* DasReq_label(const DasReq* pThis){ return pThis->sLabel; } -void DasReq_setLabel(DasReq* pThis, const char* sLabel){ _setStr(&pThis->sLabel, sLabel); } +void DasReq_setLabel(DasReq* pThis, const char* sLabel){ _dasreq_setStr(&pThis->sLabel, sLabel); } /* ************************************************************************* */ /* Entries */ -static void _freeEntry(struct das_req_entry* pE) +static void _dasreq_freeEntry(struct das_req_entry* pE) { free(pE->sKey); free(pE->sVal); @@ -174,9 +174,9 @@ int _DasReq_add(DasReq* pThis, const char* sKey, const char* sVal, bool bDerived } struct das_req_entry* pE = pThis->aEntry + pThis->uEntry; memset(pE, 0, sizeof(struct das_req_entry)); - pE->sKey = _dupz(sKey); - pE->sVal = _dupz(sVal); - if((pE->sKey == NULL)||(pE->sVal == NULL)){ _freeEntry(pE); return -1; } + pE->sKey = _dasreq_dupz(sKey); + pE->sVal = _dasreq_dupz(sVal); + if((pE->sKey == NULL)||(pE->sVal == NULL)){ _dasreq_freeEntry(pE); return -1; } pE->bDerived = bDerived; pE->nProp = -1; pThis->uEntry += 1; @@ -187,7 +187,7 @@ void _DasReq_dropDerived(DasReq* pThis) { size_t uKeep = 0; for(size_t u = 0; u < pThis->uEntry; ++u){ - if(pThis->aEntry[u].bDerived){ _freeEntry(pThis->aEntry + u); continue; } + if(pThis->aEntry[u].bDerived){ _dasreq_freeEntry(pThis->aEntry + u); continue; } if(uKeep != u) pThis->aEntry[uKeep] = pThis->aEntry[u]; ++uKeep; } @@ -200,7 +200,7 @@ void _DasReq_setVerdict(DasReq* pThis, size_t uIdx, das_req_verdict_e verdict, c struct das_req_entry* pE = pThis->aEntry + uIdx; pE->verdict = verdict; free(pE->sWhy); - pE->sWhy = (verdict == das_req_bad) ? _dupz(sWhy) : NULL; + pE->sWhy = (verdict == das_req_bad) ? _dasreq_dupz(sWhy) : NULL; } void _DasReq_clearVerdicts(DasReq* pThis) @@ -216,7 +216,7 @@ void _DasReq_clearVerdicts(DasReq* pThis) int DasReq_find(const DasReq* pThis, const char* sKey) { for(size_t u = 0; u < pThis->uEntry; ++u) - if(!pThis->aEntry[u].bDerived && _strieq(pThis->aEntry[u].sKey, sKey)) return (int)u; + if(!pThis->aEntry[u].bDerived && _dasreq_strieq(pThis->aEntry[u].sKey, sKey)) return (int)u; return -1; } @@ -233,7 +233,7 @@ DasErrCode DasReq_set(DasReq* pThis, const char* sKey, const char* sVal) int n = DasReq_find(pThis, sKey); if(n >= 0){ struct das_req_entry* pE = pThis->aEntry + n; - char* sNew = _dupz(sVal); + char* sNew = _dasreq_dupz(sVal); if(sNew == NULL) return DASERR_SRC; free(pE->sVal); pE->sVal = sNew; @@ -255,7 +255,7 @@ bool DasReq_remove(DasReq* pThis, const char* sKey) { int n = DasReq_find(pThis, sKey); if(n < 0) return false; - _freeEntry(pThis->aEntry + n); + _dasreq_freeEntry(pThis->aEntry + n); memmove(pThis->aEntry + n, pThis->aEntry + n + 1, (pThis->uEntry - n - 1) * sizeof(struct das_req_entry)); pThis->uEntry -= 1; @@ -265,7 +265,7 @@ bool DasReq_remove(DasReq* pThis, const char* sKey) void DasReq_clear(DasReq* pThis) { - for(size_t u = 0; u < pThis->uEntry; ++u) _freeEntry(pThis->aEntry + u); + for(size_t u = 0; u < pThis->uEntry; ++u) _dasreq_freeEntry(pThis->aEntry + u); pThis->uEntry = 0; for(size_t u = 0; u < pThis->uRaw; ++u){ free(pThis->aRaw[u].sName); @@ -313,7 +313,7 @@ DasErrCode DasReq_setRaw(DasReq* pThis, const char* sName, const char* sVal) } for(size_t u = 0; u < pThis->uRaw; ++u){ if(strcmp(pThis->aRaw[u].sName, sName) == 0) - return _setStr(&pThis->aRaw[u].sVal, sVal) ? DAS_OKAY : DASERR_SRC; + return _dasreq_setStr(&pThis->aRaw[u].sVal, sVal) ? DAS_OKAY : DASERR_SRC; } if(pThis->uRaw == pThis->uRawCap){ size_t uCap = pThis->uRawCap ? pThis->uRawCap * 2 : 4; @@ -325,8 +325,8 @@ DasErrCode DasReq_setRaw(DasReq* pThis, const char* sName, const char* sVal) pThis->uRawCap = uCap; } struct das_req_raw* pR = pThis->aRaw + pThis->uRaw; - pR->sName = _dupz(sName); - pR->sVal = _dupz(sVal); + pR->sName = _dasreq_dupz(sName); + pR->sVal = _dasreq_dupz(sVal); if((pR->sName == NULL)||(pR->sVal == NULL)){ free(pR->sName); free(pR->sVal); return DASERR_SRC; @@ -350,7 +350,7 @@ const char* DasReq_raw(const DasReq* pThis, size_t u, const char** psVal) typedef struct { char* s; size_t u; size_t uCap; bool bFail; } _out_t; -static void _put(_out_t* pO, const char* s) +static void _dasreq_put(_out_t* pO, const char* s) { size_t n = strlen(s); if(pO->u + n + 1 > pO->uCap){ @@ -365,54 +365,54 @@ static void _put(_out_t* pO, const char* s) pO->u += n; } -static void _putStr(_out_t* pO, const char* s) +static void _dasreq_putStr(_out_t* pO, const char* s) { - _put(pO, "\""); + _dasreq_put(pO, "\""); char sEsc[8]; for(const unsigned char* p = (const unsigned char*)s; *p; ++p){ switch(*p){ - case '"': _put(pO, "\\\""); break; - case '\\': _put(pO, "\\\\"); break; - case '\n': _put(pO, "\\n"); break; - case '\r': _put(pO, "\\r"); break; - case '\t': _put(pO, "\\t"); break; + case '"': _dasreq_put(pO, "\\\""); break; + case '\\': _dasreq_put(pO, "\\\\"); break; + case '\n': _dasreq_put(pO, "\\n"); break; + case '\r': _dasreq_put(pO, "\\r"); break; + case '\t': _dasreq_put(pO, "\\t"); break; default: - if(*p < 0x20){ snprintf(sEsc, sizeof(sEsc), "\\u%04x", *p); _put(pO, sEsc); } - else{ sEsc[0] = (char)*p; sEsc[1] = '\0'; _put(pO, sEsc); } + if(*p < 0x20){ snprintf(sEsc, sizeof(sEsc), "\\u%04x", *p); _dasreq_put(pO, sEsc); } + else{ sEsc[0] = (char)*p; sEsc[1] = '\0'; _dasreq_put(pO, sEsc); } } } - _put(pO, "\""); + _dasreq_put(pO, "\""); } -static void _putPair(_out_t* pO, const char* sKey, const char* sVal, bool bFirst) +static void _dasreq_putPair(_out_t* pO, const char* sKey, const char* sVal, bool bFirst) { - if(!bFirst) _put(pO, ",\n"); - _put(pO, " "); - _putStr(pO, sKey); - _put(pO, ": "); - _putStr(pO, sVal); + if(!bFirst) _dasreq_put(pO, ",\n"); + _dasreq_put(pO, " "); + _dasreq_putStr(pO, sKey); + _dasreq_put(pO, ": "); + _dasreq_putStr(pO, sVal); } char* DasReq_text(const DasReq* pThis) { _out_t o = {NULL, 0, 0, false}; - _put(&o, "{\n \"source\": {\n"); - _putPair(&o, "path", pThis->sPath, true); - _putPair(&o, "url", pThis->sUrl, false); - _putPair(&o, "version", pThis->sVersion, false); - _put(&o, "\n },\n \"label\": "); - _putStr(&o, pThis->sLabel); - _put(&o, ",\n \"settings\": {\n"); + _dasreq_put(&o, "{\n \"source\": {\n"); + _dasreq_putPair(&o, "path", pThis->sPath, true); + _dasreq_putPair(&o, "url", pThis->sUrl, false); + _dasreq_putPair(&o, "version", pThis->sVersion, false); + _dasreq_put(&o, "\n },\n \"label\": "); + _dasreq_putStr(&o, pThis->sLabel); + _dasreq_put(&o, ",\n \"settings\": {\n"); bool bFirst = true; for(size_t u = 0; u < pThis->uEntry; ++u){ if(pThis->aEntry[u].bDerived) continue; - _putPair(&o, pThis->aEntry[u].sKey, pThis->aEntry[u].sVal, bFirst); + _dasreq_putPair(&o, pThis->aEntry[u].sKey, pThis->aEntry[u].sVal, bFirst); bFirst = false; } - _put(&o, "\n },\n \"params\": {\n"); + _dasreq_put(&o, "\n },\n \"params\": {\n"); for(size_t u = 0; u < pThis->uRaw; ++u) - _putPair(&o, pThis->aRaw[u].sName, pThis->aRaw[u].sVal, u == 0); - _put(&o, "\n }\n}\n"); + _dasreq_putPair(&o, pThis->aRaw[u].sName, pThis->aRaw[u].sVal, u == 0); + _dasreq_put(&o, "\n }\n}\n"); if(o.bFail){ free(o.s); return NULL; } return o.s; } @@ -445,7 +445,7 @@ DasErrCode DasReq_write(const DasReq* pThis, const char* sPath) the dotted key, so a slash path is folded to a key on the way in: drop "props", join with dots. Section aliases are left for the source's matcher, which accepts them anyway. */ -static void _pathToKey(const char* sPath, char* sKey, size_t uLen) +static void _dasreq_pathToKey(const char* sPath, char* sKey, size_t uLen) { size_t u = 0; const char* pBeg = sPath; @@ -465,7 +465,7 @@ static void _pathToKey(const char* sPath, char* sKey, size_t uLen) } } -static const char* _jstr(const DasJdo* p, const char* sPath) +static const char* _dasreq_jstr(const DasJdo* p, const char* sPath) { const DasJdo* pSub = p ? DasJdo_get(p, sPath) : NULL; return ((pSub != NULL)&&(pSub->type == das_json_type_str)) ? DasJdo_string(pSub) : NULL; @@ -483,14 +483,14 @@ DasReq* new_DasReq_text(const char* sJson, size_t uLen) if(pThis == NULL){ free(pDoc); return NULL; } const DasJdo* pSrc = DasJdo_get(pDoc, "source"); - DasReq_setSrc(pThis, _jstr(pSrc, "path"), _jstr(pSrc, "url"), _jstr(pSrc, "version")); - DasReq_setLabel(pThis, _jstr(pDoc, "label")); + DasReq_setSrc(pThis, _dasreq_jstr(pSrc, "path"), _dasreq_jstr(pSrc, "url"), _dasreq_jstr(pSrc, "version")); + DasReq_setLabel(pThis, _dasreq_jstr(pDoc, "label")); char sKey[256], sVal[1024]; bool bOk = true; const DasJdo* pSet = DasJdo_get(pDoc, "settings"); for(const das_json_dict_el* pEl = DasJdo_dictFirst(pSet); bOk && pEl; pEl = pEl->next){ - if(strchr(pEl->name->string, '/')) _pathToKey(pEl->name->string, sKey, sizeof(sKey)); + if(strchr(pEl->name->string, '/')) _dasreq_pathToKey(pEl->name->string, sKey, sizeof(sKey)); else{ strncpy(sKey, pEl->name->string, sizeof(sKey) - 1); sKey[sizeof(sKey)-1] = '\0'; } if(!_das_json_scalar(pEl->value, sVal, sizeof(sVal))) sVal[0] = '\0'; bOk = (DasReq_set(pThis, sKey, sVal) == DAS_OKAY); diff --git a/das3/source.c b/das3/source.c index 0428e04..229aa75 100644 --- a/das3/source.c +++ b/das3/source.c @@ -26,7 +26,7 @@ * Grew out of the pilot in utilities/das_fetch.c, which carried the * translation rules for two node layouts. Those rules are not written * down anywhere else yet; until the catalog spec states them, the - * comments on _translate() are the reference. + * comments on _DasSrcCtrl_translate() are the reference. */ #define _POSIX_C_SOURCE 200112L @@ -57,7 +57,7 @@ static const char* g_sSecName[das_src_sec_size] = {"coords", "data", "formats", /* Small string and JSON helpers. DasJdo_get() takes slash paths, so the * dots in parameter names such as "read.time.min" are not special to it. */ -static bool _strieq(const char* a, const char* b) +static bool _dassrc_strieq(const char* a, const char* b) { if((a == NULL)||(b == NULL)) return false; while(*a && *b){ @@ -67,40 +67,40 @@ static bool _strieq(const char* a, const char* b) return (*a == '\0') && (*b == '\0'); } -static void _strcpyz(char* sDest, const char* sSrc, size_t uDest) +static void _dassrc_strcpyz(char* sDest, const char* sSrc, size_t uDest) { if(uDest == 0) return; strncpy(sDest, sSrc ? sSrc : "", uDest - 1); sDest[uDest - 1] = '\0'; } -static const DasJdo* _jget(const DasJdo* p, const char* sPath) +static const DasJdo* _dassrc_jget(const DasJdo* p, const char* sPath) { return ((p == NULL)||(sPath == NULL)) ? NULL : DasJdo_get(p, sPath); } -static const char* _jstr(const DasJdo* p, const char* sPath) +static const char* _dassrc_jstr(const DasJdo* p, const char* sPath) { - const DasJdo* pSub = _jget(p, sPath); + const DasJdo* pSub = _dassrc_jget(p, sPath); return ((pSub != NULL)&&(pSub->type == das_json_type_str)) ? DasJdo_string(pSub) : NULL; } -static bool _jisdict(const DasJdo* p){ return (p != NULL)&&(p->type == das_json_type_dict); } -static bool _jisary(const DasJdo* p){ return (p != NULL)&&(p->type == das_json_type_ary); } -static bool _jtrue(const DasJdo* p){ return (p != NULL)&&(p->type == das_json_type_true); } -static bool _jisbool(const DasJdo* p){ +static bool _dassrc_jisdict(const DasJdo* p){ return (p != NULL)&&(p->type == das_json_type_dict); } +static bool _dassrc_jisary(const DasJdo* p){ return (p != NULL)&&(p->type == das_json_type_ary); } +static bool _dassrc_jtrue(const DasJdo* p){ return (p != NULL)&&(p->type == das_json_type_true); } +static bool _dassrc_jisbool(const DasJdo* p){ return (p != NULL)&&((p->type == das_json_type_true)||(p->type == das_json_type_false)); } -static size_t _jlen(const DasJdo* p) +static size_t _dassrc_jlen(const DasJdo* p) { - if(_jisdict(p)) return ((const das_json_dict*)p->value)->length; - if(_jisary(p)) return ((const das_json_ary*)p->value)->length; + if(_dassrc_jisdict(p)) return ((const das_json_dict*)p->value)->length; + if(_dassrc_jisary(p)) return ((const das_json_ary*)p->value)->length; return 0; } /* das_strdup() returns NULL for "", which here would turn an empty default into no default. Empty strings are copied as empty strings. */ -static char* _strdupz(const char* s) +static char* _dassrc_strdupz(const char* s) { if((s == NULL)||(s[0] == '\0')) return (char*)calloc(1, 1); return das_strdup(s); @@ -115,25 +115,25 @@ static char* _strdupz(const char* s) * arena was considered and rejected: it needs an exact size pass that * duplicates the formatting logic below, and it hides overruns from * valgrind. Revisit if a heap profile of a browser build points here. */ -static char* _scalarDup(const DasJdo* p) +static char* _dassrc_scalarDup(const DasJdo* p) { char sBuf[SCALAR_SZ]; if(!_das_json_scalar(p, sBuf, sizeof(sBuf))) return NULL; - return _strdupz(sBuf); + return _dassrc_strdupz(sBuf); } -static bool _parseBool(const char* s, bool* pOut) +static bool _dassrc_parseBool(const char* s, bool* pOut) { if(s == NULL) return false; - if(_strieq(s,"true")||_strieq(s,"yes")||_strieq(s,"on")||_strieq(s,"1")|| - _strieq(s,"t")||_strieq(s,"y")){ *pOut = true; return true; } - if(_strieq(s,"false")||_strieq(s,"no")||_strieq(s,"off")||_strieq(s,"0")|| - _strieq(s,"f")||_strieq(s,"n")){ *pOut = false; return true; } + if(_dassrc_strieq(s,"true")||_dassrc_strieq(s,"yes")||_dassrc_strieq(s,"on")||_dassrc_strieq(s,"1")|| + _dassrc_strieq(s,"t")||_dassrc_strieq(s,"y")){ *pOut = true; return true; } + if(_dassrc_strieq(s,"false")||_dassrc_strieq(s,"no")||_dassrc_strieq(s,"off")||_dassrc_strieq(s,"0")|| + _dassrc_strieq(s,"f")||_dassrc_strieq(s,"n")){ *pOut = false; return true; } return false; } /* snprintf that appends and never runs the cursor past the buffer */ -static size_t _catf(char* sBuf, size_t uLen, size_t u, const char* sFmt, ...) +static size_t _dassrc_catf(char* sBuf, size_t uLen, size_t u, const char* sFmt, ...) { if(u >= uLen) return u; va_list ap; @@ -154,14 +154,14 @@ const char* das_src_sec_name(das_src_sec_e sec) bool _das_src_secByName(const char* s, das_src_sec_e* pSec) { - if(_strieq(s,"coord")||_strieq(s,"coords")||_strieq(s,"coordinate")||_strieq(s,"coordinates")){ + if(_dassrc_strieq(s,"coord")||_dassrc_strieq(s,"coords")||_dassrc_strieq(s,"coordinate")||_dassrc_strieq(s,"coordinates")){ *pSec = das_src_sec_coord; return true; } - if(_strieq(s,"data")){ *pSec = das_src_sec_data; return true; } - if(_strieq(s,"format")||_strieq(s,"formats")||_strieq(s,"fmt")){ + if(_dassrc_strieq(s,"data")){ *pSec = das_src_sec_data; return true; } + if(_dassrc_strieq(s,"format")||_dassrc_strieq(s,"formats")||_dassrc_strieq(s,"fmt")){ *pSec = das_src_sec_format; return true; } - if(_strieq(s,"option")||_strieq(s,"options")||_strieq(s,"opt")||_strieq(s,"opts")){ + if(_dassrc_strieq(s,"option")||_dassrc_strieq(s,"options")||_dassrc_strieq(s,"opt")||_dassrc_strieq(s,"opts")){ *pSec = das_src_sec_option; return true; } return false; @@ -170,17 +170,17 @@ bool _das_src_secByName(const char* s, das_src_sec_e* pSec) const char* _das_src_propName(const char* s) { if(s == NULL) return ""; - if(_strieq(s, "minimum")||_strieq(s, "min")) return "min"; - if(_strieq(s, "maximum")||_strieq(s, "max")) return "max"; - if(_strieq(s, "resolution")||_strieq(s, "res")) return "res"; - if(_strieq(s, "interval")||_strieq(s, "inter")) return "inter"; - if(_strieq(s, "enable")||_strieq(s, "enabled")) return "enabled"; + if(_dassrc_strieq(s, "minimum")||_dassrc_strieq(s, "min")) return "min"; + if(_dassrc_strieq(s, "maximum")||_dassrc_strieq(s, "max")) return "max"; + if(_dassrc_strieq(s, "resolution")||_dassrc_strieq(s, "res")) return "res"; + if(_dassrc_strieq(s, "interval")||_dassrc_strieq(s, "inter")) return "inter"; + if(_dassrc_strieq(s, "enable")||_dassrc_strieq(s, "enabled")) return "enabled"; return s; } bool _das_src_propEq(const char* a, const char* b) { - return _strieq(_das_src_propName(a), _das_src_propName(b)); + return _dassrc_strieq(_das_src_propName(a), _das_src_propName(b)); } int _das_src_splitKey(const char* sKey, char aParts[3][DAS_MAX_ID_BUFSZ]) @@ -206,19 +206,19 @@ int _das_src_splitKey(const char* sKey, char aParts[3][DAS_MAX_ID_BUFSZ]) /* Fold a type word from a node to the enum. NULL and unknown words are * text, the caller decides whether that is final. */ -static bool _typeByName(const char* s, das_srcprop_type_e* pType) +static bool _dassrc_typeByName(const char* s, das_srcprop_type_e* pType) { if(s == NULL) return false; - if(_strieq(s,"boolean")||_strieq(s,"bool")){ *pType = das_srcprop_bool; return true; } - if(_strieq(s,"integer")||_strieq(s,"int")){ *pType = das_srcprop_int; return true; } - if(_strieq(s,"real")||_strieq(s,"double")||_strieq(s,"float")){ + if(_dassrc_strieq(s,"boolean")||_dassrc_strieq(s,"bool")){ *pType = das_srcprop_bool; return true; } + if(_dassrc_strieq(s,"integer")||_dassrc_strieq(s,"int")){ *pType = das_srcprop_int; return true; } + if(_dassrc_strieq(s,"real")||_dassrc_strieq(s,"double")||_dassrc_strieq(s,"float")){ *pType = das_srcprop_real; return true; } - if(_strieq(s,"isotime")||_strieq(s,"time")||_strieq(s,"datetime")){ + if(_dassrc_strieq(s,"isotime")||_dassrc_strieq(s,"time")||_dassrc_strieq(s,"datetime")){ *pType = das_srcprop_time; return true; } - if(_strieq(s,"enum")){ *pType = das_srcprop_enum; return true; } - if(_strieq(s,"string")||_strieq(s,"text")){ *pType = das_srcprop_text; return true; } + if(_dassrc_strieq(s,"enum")){ *pType = das_srcprop_enum; return true; } + if(_dassrc_strieq(s,"string")||_dassrc_strieq(s,"text")){ *pType = das_srcprop_text; return true; } return false; } @@ -228,22 +228,22 @@ static bool _typeByName(const char* s, das_srcprop_type_e* pType) * "string" with a list of choices is still a list of choices to anyone * presenting it. A flag set parameter type is a server side packing * detail and says nothing about the single toggle that feeds it. */ -static void _resolveType(DasSrcProp* p) +static void _DasSrcProp_resolveType(DasSrcProp* p) { - p->sTypeName = _jstr(p->pJdo, "type"); + p->sTypeName = _dassrc_jstr(p->pJdo, "type"); if(p->sTypeName == NULL) p->sTypeName = ""; const DasSrcCtrl* pC = p->pCtrl; - const DasJdo* pVal = _jget(p->pJdo, "value"); + const DasJdo* pVal = _dassrc_jget(p->pJdo, "value"); if(pC && (pC->uEnum > 0)){ p->type = das_srcprop_enum; return; } - if(_typeByName(p->sTypeName, &(p->type))){ + if(_dassrc_typeByName(p->sTypeName, &(p->type))){ if(p->type != das_srcprop_enum) return; } - if(_jisbool(pVal) || (pC && (_jstr(pC->pSet, "flag") != NULL))){ + if(_dassrc_jisbool(pVal) || (pC && (_dassrc_jstr(pC->pSet, "flag") != NULL))){ p->type = das_srcprop_bool; return; } - const char* sParamType = pC ? _jstr(pC->pParam, "type") : NULL; - if(_typeByName(sParamType, &(p->type)) && (p->type != das_srcprop_enum)){ + const char* sParamType = pC ? _dassrc_jstr(pC->pParam, "type") : NULL; + if(_dassrc_typeByName(sParamType, &(p->type)) && (p->type != das_srcprop_enum)){ if(p->sTypeName[0] == '\0') p->sTypeName = sParamType; return; } @@ -253,19 +253,19 @@ static void _resolveType(DasSrcProp* p) /* Enum entries are usually dicts carrying value plus per item pval or flag * overrides, but a bare scalar list is accepted too. */ -static bool _fillEnum(DasSrcCtrl* p) +static bool _DasSrcCtrl_fillEnum(DasSrcCtrl* p) { - const DasJdo* pEnum = _jget(p->pSet, "enum"); - if(!_jisary(pEnum)) return true; + const DasJdo* pEnum = _dassrc_jget(p->pSet, "enum"); + if(!_dassrc_jisary(pEnum)) return true; - size_t uMax = _jlen(pEnum); + size_t uMax = _dassrc_jlen(pEnum); if(uMax == 0) return true; p->aEnum = (char**)calloc(uMax, sizeof(char*)); if(p->aEnum == NULL) return false; for(const das_json_ary_el* pEl = DasJdo_aryFirst(pEnum); pEl; pEl = pEl->next){ - const DasJdo* pVal = _jisdict(pEl->value) ? _jget(pEl->value, "value") : pEl->value; - char* s = _scalarDup(pVal); + const DasJdo* pVal = _dassrc_jisdict(pEl->value) ? _dassrc_jget(pEl->value, "value") : pEl->value; + char* s = _dassrc_scalarDup(pVal); if(s == NULL) continue; p->aEnum[p->uEnum] = s; p->uEnum += 1; @@ -273,24 +273,24 @@ static bool _fillEnum(DasSrcCtrl* p) return true; } -static void _fillRange(DasSrcCtrl* p) +static void _DasSrcCtrl_fillRange(DasSrcCtrl* p) { const DasSrcProp* pProp = p->pProp; - const DasJdo* pRange = _jget(p->pSet, "range"); - if(!_jisary(pRange)) pRange = _jget(p->pParam, "range"); - if(!_jisary(pRange) && (pProp->sec == das_src_sec_coord) && + const DasJdo* pRange = _dassrc_jget(p->pSet, "range"); + if(!_dassrc_jisary(pRange)) pRange = _dassrc_jget(p->pParam, "range"); + if(!_dassrc_jisary(pRange) && (pProp->sec == das_src_sec_coord) && (_das_src_propEq(pProp->sName, "min") || _das_src_propEq(pProp->sName, "max"))) - pRange = _jget(pProp->pGroup->pJdo, "validRange"); - if(!_jisary(pRange)) return; + pRange = _dassrc_jget(pProp->pGroup->pJdo, "validRange"); + if(!_dassrc_jisary(pRange)) return; const das_json_ary_el* pEl = DasJdo_aryFirst(pRange); if(pEl == NULL) return; - p->sLo = _scalarDup(pEl->value); - if(pEl->next) p->sHi = _scalarDup(pEl->next->value); + p->sLo = _dassrc_scalarDup(pEl->value); + if(pEl->next) p->sHi = _dassrc_scalarDup(pEl->next->value); } /* Release what a control owns; the control itself lives in aCtrl */ -static void _clearCtrl(DasSrcCtrl* p) +static void _DasSrcCtrl_clear(DasSrcCtrl* p) { if(p == NULL) return; for(size_t v = 0; v < p->uEnum; ++v) free(p->aEnum[v]); @@ -304,16 +304,16 @@ static void _clearCtrl(DasSrcCtrl* p) * 2018 layout properties sit beside the group's name and title * strings, and a units dict with a value counts as the units property * just as it does under props today. */ -static bool _isPropDict(const DasJdo* p) +static bool _dassrc_isPropDict(const DasJdo* p) { - if(!_jisdict(p)) return false; - return (_jget(p, "value") != NULL)||(_jget(p, "set") != NULL)||(_jget(p, "type") != NULL); + if(!_dassrc_jisdict(p)) return false; + return (_dassrc_jget(p, "value") != NULL)||(_dassrc_jget(p, "set") != NULL)||(_dassrc_jget(p, "type") != NULL); } -static const DasJdo* _groupProps(const DasJdo* pGroup) +static const DasJdo* _dassrc_groupProps(const DasJdo* pGroup) { - const DasJdo* pProps = _jget(pGroup, "props"); - return _jisdict(pProps) ? pProps : pGroup; + const DasJdo* pProps = _dassrc_jget(pGroup, "props"); + return _dassrc_jisdict(pProps) ? pProps : pGroup; } /* Count and fill share one walk. With pSrc NULL only the counts are @@ -326,17 +326,17 @@ typedef struct { /* A property is settable when its set block names a protocol parameter. * The counting pass and the fill pass must agree on this, so it is one * function. */ -static const DasJdo* _setBlock(const DasJdo* pJdo) +static const DasJdo* _dassrc_setBlock(const DasJdo* pJdo) { - const DasJdo* pSet = _jget(pJdo, "set"); - return (_jisdict(pSet) && (_jstr(pSet, "param") != NULL)) ? pSet : NULL; + const DasJdo* pSet = _dassrc_jget(pJdo, "set"); + return (_dassrc_jisdict(pSet) && (_dassrc_jstr(pSet, "param") != NULL)) ? pSet : NULL; } -static void _walkProp(_walk_t* pW, DasSrcGroup* pG, const char* sName, const DasJdo* pJdo) +static void _dassrc_walkProp(_walk_t* pW, DasSrcGroup* pG, const char* sName, const DasJdo* pJdo) { if(pW->pSrc == NULL){ pW->uProp += 1; - if(_setBlock(pJdo) != NULL) pW->uCtrl += 1; + if(_dassrc_setBlock(pJdo) != NULL) pW->uCtrl += 1; return; } if(pW->bFail) return; @@ -357,11 +357,11 @@ static void _walkProp(_walk_t* pW, DasSrcGroup* pG, const char* sName, const Das else snprintf(p->sKey, uKey, "%s.%s.%s", g_sSecName[pG->sec], pG->sName, sName); - p->sValue = _scalarDup(_jget(pJdo, "value")); + p->sValue = _dassrc_scalarDup(_dassrc_jget(pJdo, "value")); - const DasJdo* pSet = _jget(pJdo, "set"); - if(_jisdict(pSet)){ - const char* sParam = _jstr(pSet, "param"); + const DasJdo* pSet = _dassrc_jget(pJdo, "set"); + if(_dassrc_jisdict(pSet)){ + const char* sParam = _dassrc_jstr(pSet, "param"); if(sParam == NULL){ daslog_warn_v("Property %s has a set block that names no parameter, " "treating it as fixed", p->sKey); @@ -375,26 +375,26 @@ static void _walkProp(_walk_t* pW, DasSrcGroup* pG, const char* sName, const Das pC->pProp = p; pC->pSet = pSet; pC->sParam = sParam; - pC->pParam = _jget(pW->pParams, sParam); - pC->bRequired = _jtrue(_jget(pSet, "required")) || _jtrue(_jget(pC->pParam, "required")); + pC->pParam = _dassrc_jget(pW->pParams, sParam); + pC->bRequired = _dassrc_jtrue(_dassrc_jget(pSet, "required")) || _dassrc_jtrue(_dassrc_jget(pC->pParam, "required")); p->pCtrl = pC; - if(!_fillEnum(pC)){ pW->bFail = true; return; } - _fillRange(pC); + if(!_DasSrcCtrl_fillEnum(pC)){ pW->bFail = true; return; } + _DasSrcCtrl_fillRange(pC); } } - _resolveType(p); + _DasSrcProp_resolveType(p); if(_das_src_propEq(sName, "enabled") && (pG->nEnable < 0)){ pG->nEnable = (int)(pW->uProp - pG->uFirst); - pG->sXor = _jstr(pJdo, "xorGroup"); + pG->sXor = _dassrc_jstr(pJdo, "xorGroup"); } pW->uProp += 1; pG->uCount += 1; } -static void _walkGroup(_walk_t* pW, das_src_sec_e sec, const char* sName, const DasJdo* pJdo) +static void _dassrc_walkGroup(_walk_t* pW, das_src_sec_e sec, const char* sName, const DasJdo* pJdo) { - if(!_jisdict(pJdo)) return; + if(!_dassrc_jisdict(pJdo)) return; DasSrcGroup* pG = NULL; if(pW->pSrc != NULL){ pG = pW->pSrc->aGroup + pW->uGroup; @@ -406,9 +406,9 @@ static void _walkGroup(_walk_t* pW, das_src_sec_e sec, const char* sName, const pG->uFirst = pW->uProp; pG->nEnable = -1; - const DasJdo* pMime = _jget(pJdo, "mimeTypes"); - if(_jisary(pMime) && (_jlen(pMime) > 0)){ - pG->aMime = (const char**)calloc(_jlen(pMime), sizeof(char*)); + const DasJdo* pMime = _dassrc_jget(pJdo, "mimeTypes"); + if(_dassrc_jisary(pMime) && (_dassrc_jlen(pMime) > 0)){ + pG->aMime = (const char**)calloc(_dassrc_jlen(pMime), sizeof(char*)); if(pG->aMime == NULL){ pW->bFail = true; return; } for(const das_json_ary_el* pEl = DasJdo_aryFirst(pMime); pEl; pEl = pEl->next){ if(DasJdo_string(pEl->value) == NULL) continue; @@ -420,43 +420,43 @@ static void _walkGroup(_walk_t* pW, das_src_sec_e sec, const char* sName, const } pW->uGroup += 1; - const DasJdo* pProps = _groupProps(pJdo); + const DasJdo* pProps = _dassrc_groupProps(pJdo); for(const das_json_dict_el* pEl = DasJdo_dictFirst(pProps); pEl; pEl = pEl->next){ - if(!_isPropDict(pEl->value)) continue; - _walkProp(pW, pG, pEl->name->string, pEl->value); + if(!_dassrc_isPropDict(pEl->value)) continue; + _dassrc_walkProp(pW, pG, pEl->name->string, pEl->value); } } /* Groups of a section in presentation order: "time" first for coordinates, * otherwise as the node lists them. Forms want time on top and nodes do * not always put it there. */ -static void _walkSection(_walk_t* pW, das_src_sec_e sec, const DasJdo* pSec) +static void _dassrc_walkSection(_walk_t* pW, das_src_sec_e sec, const DasJdo* pSec) { - if(!_jisdict(pSec)) return; + if(!_dassrc_jisdict(pSec)) return; if(pW->pSrc != NULL) pW->pSrc->aSecFirst[sec] = pW->uGroup; if(sec == das_src_sec_coord){ for(const das_json_dict_el* pEl = DasJdo_dictFirst(pSec); pEl; pEl = pEl->next) - if(_strieq(pEl->name->string, "time")) _walkGroup(pW, sec, pEl->name->string, pEl->value); + if(_dassrc_strieq(pEl->name->string, "time")) _dassrc_walkGroup(pW, sec, pEl->name->string, pEl->value); } for(const das_json_dict_el* pEl = DasJdo_dictFirst(pSec); pEl; pEl = pEl->next){ - if((sec == das_src_sec_coord) && _strieq(pEl->name->string, "time")) continue; - _walkGroup(pW, sec, pEl->name->string, pEl->value); + if((sec == das_src_sec_coord) && _dassrc_strieq(pEl->name->string, "time")) continue; + _dassrc_walkGroup(pW, sec, pEl->name->string, pEl->value); } } -static void _walkIface(_walk_t* pW, const DasJdo* pIface) +static void _dassrc_walkIface(_walk_t* pW, const DasJdo* pIface) { - const DasJdo* pCoords = _jget(pIface, "coords"); - if(pCoords == NULL) pCoords = _jget(pIface, "coordinates"); - _walkSection(pW, das_src_sec_coord, pCoords); - _walkSection(pW, das_src_sec_data, _jget(pIface, "data")); - _walkSection(pW, das_src_sec_format, _jget(pIface, "formats")); + const DasJdo* pCoords = _dassrc_jget(pIface, "coords"); + if(pCoords == NULL) pCoords = _dassrc_jget(pIface, "coordinates"); + _dassrc_walkSection(pW, das_src_sec_coord, pCoords); + _dassrc_walkSection(pW, das_src_sec_data, _dassrc_jget(pIface, "data")); + _dassrc_walkSection(pW, das_src_sec_format, _dassrc_jget(pIface, "formats")); /* Options are one group with no enclosing section */ - const DasJdo* pOpts = _jget(pIface, "options"); - if(_jisdict(pOpts)){ + const DasJdo* pOpts = _dassrc_jget(pIface, "options"); + if(_dassrc_jisdict(pOpts)){ if(pW->pSrc != NULL) pW->pSrc->aSecFirst[das_src_sec_option] = pW->uGroup; - _walkGroup(pW, das_src_sec_option, "", pOpts); + _dassrc_walkGroup(pW, das_src_sec_option, "", pOpts); } } @@ -466,7 +466,7 @@ static void _walkIface(_walk_t* pW, const DasJdo* pIface) /* "coords/time/props/min" -> "coords.time.min". The section token is * folded to the current spelling so a saved requisition reads the same * whatever node it came from. */ -static bool _exPathToKey(const char* sPath, char* sKey, size_t uLen) +static bool _dassrc_exPathToKey(const char* sPath, char* sKey, size_t uLen) { size_t u = 0; int nTok = 0; @@ -484,7 +484,7 @@ static bool _exPathToKey(const char* sPath, char* sKey, size_t uLen) else if((nTok == 0) && _das_src_secByName(sTok, &sec)) sOut = g_sSecName[sec]; if(sOut != NULL){ if(u + strlen(sOut) + 2 >= uLen) return false; - u = _catf(sKey, uLen, u, "%s%s", nTok ? "." : "", sOut); + u = _dassrc_catf(sKey, uLen, u, "%s%s", nTok ? "." : "", sOut); ++nTok; } if(pEnd == NULL) break; @@ -495,12 +495,12 @@ static bool _exPathToKey(const char* sPath, char* sKey, size_t uLen) /* The flag value a control sends, without any prefix: the parameter's * flags entry for it, or the flag name itself */ -static const char* _ctrlFlagVal(const DasSrcCtrl* pC, const char* sFlag, char* sBuf, size_t uLen) +static const char* _DasSrcCtrl_flagVal(const DasSrcCtrl* pC, const char* sFlag, char* sBuf, size_t uLen) { char sPath[DAS_MAX_ID_BUFSZ + 8]; snprintf(sPath, sizeof(sPath), "flags/%s", sFlag); - const DasJdo* pF = _jget(pC->pParam, sPath); - if(!_das_json_scalar(_jget(pF, "value"), sBuf, uLen)) _strcpyz(sBuf, sFlag, uLen); + const DasJdo* pF = _dassrc_jget(pC->pParam, sPath); + if(!_das_json_scalar(_dassrc_jget(pF, "value"), sBuf, uLen)) _dassrc_strcpyz(sBuf, sFlag, uLen); return sBuf; } @@ -508,7 +508,7 @@ static const char* _ctrlFlagVal(const DasSrcCtrl* pC, const char* sFlag, char* s * the control that would have produced it, so the requisition looks the * same as one from a current node. Parameters no control explains are * kept as raw parameters, nothing is dropped. */ -static bool _exampleFromParams(DasSrc* pThis, DasReq* pReq, const DasJdo* pParams) +static bool _DasSrc_exampleFromParams(DasSrc* pThis, DasReq* pReq, const DasJdo* pParams) { char sVal[SCALAR_SZ], sFlagVal[SCALAR_SZ], sPVal[SCALAR_SZ]; for(const das_json_dict_el* pP = DasJdo_dictFirst(pParams); pP; pP = pP->next){ @@ -520,7 +520,7 @@ static bool _exampleFromParams(DasSrc* pThis, DasReq* pReq, const DasJdo* pParam for(size_t u = 0; u < pThis->uProp; ++u){ const DasSrcCtrl* pC = pThis->aProp[u].pCtrl; if((pC == NULL)||(strcmp(pC->sParam, sName) != 0)) continue; - if(_jstr(pC->pSet, "flag") != NULL){ bFlagSet = true; break; } + if(_dassrc_jstr(pC->pSet, "flag") != NULL){ bFlagSet = true; break; } } if(bFlagSet){ @@ -528,8 +528,8 @@ static bool _exampleFromParams(DasSrc* pThis, DasReq* pReq, const DasJdo* pParam for(size_t u = 0; (sSep == NULL) && (u < pThis->uProp); ++u){ const DasSrcCtrl* pC = pThis->aProp[u].pCtrl; if((pC == NULL)||(strcmp(pC->sParam, sName) != 0)) continue; - sSep = _jstr(pC->pParam, "flagSep"); - if(sSep == NULL) sSep = _jstr(pC->pParam, "flag_sep"); + sSep = _dassrc_jstr(pC->pParam, "flagSep"); + if(sSep == NULL) sSep = _dassrc_jstr(pC->pParam, "flag_sep"); } if((sSep == NULL)||(sSep[0] == '\0')) sSep = " "; @@ -542,10 +542,10 @@ static bool _exampleFromParams(DasSrc* pThis, DasReq* pReq, const DasJdo* pParam for(size_t u = 0; !bFound && (u < pThis->uProp); ++u){ const DasSrcCtrl* pC = pThis->aProp[u].pCtrl; if((pC == NULL)||(strcmp(pC->sParam, sName) != 0)) continue; - const char* sFlag = _jstr(pC->pSet, "flag"); + const char* sFlag = _dassrc_jstr(pC->pSet, "flag"); if(sFlag == NULL) continue; - _ctrlFlagVal(pC, sFlag, sFlagVal, sizeof(sFlagVal)); - if(_strieq(sFlagVal, pTok)){ + _DasSrcCtrl_flagVal(pC, sFlag, sFlagVal, sizeof(sFlagVal)); + if(_dassrc_strieq(sFlagVal, pTok)){ if(DasReq_set(pReq, pThis->aProp[u].sKey, "true") != DAS_OKAY) return false; bFound = true; } @@ -566,15 +566,15 @@ static bool _exampleFromParams(DasSrc* pThis, DasReq* pReq, const DasJdo* pParam const DasSrcCtrl* pC = p->pCtrl; if((pC == NULL)||(strcmp(pC->sParam, sName) != 0)) continue; - const DasJdo* pEnum = _jget(pC->pSet, "enum"); - if(_jisary(pEnum)){ + const DasJdo* pEnum = _dassrc_jget(pC->pSet, "enum"); + if(_dassrc_jisary(pEnum)){ /* which choice sends this protocol value? */ for(const das_json_ary_el* pEl = DasJdo_aryFirst(pEnum); !bHandled && pEl; pEl = pEl->next){ - const DasJdo* pItem = _jisdict(pEl->value) ? _jget(pEl->value, "value") : pEl->value; + const DasJdo* pItem = _dassrc_jisdict(pEl->value) ? _dassrc_jget(pEl->value, "value") : pEl->value; char sItem[SCALAR_SZ]; if(!_das_json_scalar(pItem, sItem, sizeof(sItem))) continue; - bool bHit = _das_json_scalar(_jget(pEl->value, "pval"), sPVal, sizeof(sPVal)) ? - _strieq(sPVal, sVal) : _strieq(sItem, sVal); + bool bHit = _das_json_scalar(_dassrc_jget(pEl->value, "pval"), sPVal, sizeof(sPVal)) ? + _dassrc_strieq(sPVal, sVal) : _dassrc_strieq(sItem, sVal); if(bHit){ if(DasReq_set(pReq, p->sKey, sItem) != DAS_OKAY) return false; bHandled = true; @@ -582,12 +582,12 @@ static bool _exampleFromParams(DasSrc* pThis, DasReq* pReq, const DasJdo* pParam } continue; } - if(_das_json_scalar(_jget(pC->pSet, "pval"), sPVal, sizeof(sPVal))){ + if(_das_json_scalar(_dassrc_jget(pC->pSet, "pval"), sPVal, sizeof(sPVal))){ /* a fixed protocol value stands for one state of a toggle */ - if(_strieq(sPVal, sVal)){ + if(_dassrc_strieq(sPVal, sVal)){ bool bOn = true; - const DasJdo* pTrig = _jget(pC->pSet, "value"); - if(_jisbool(pTrig)) bOn = _jtrue(pTrig); + const DasJdo* pTrig = _dassrc_jget(pC->pSet, "value"); + if(_dassrc_jisbool(pTrig)) bOn = _dassrc_jtrue(pTrig); if(DasReq_set(pReq, p->sKey, bOn ? "true" : "false") != DAS_OKAY) return false; bHandled = true; } @@ -602,17 +602,17 @@ static bool _exampleFromParams(DasSrc* pThis, DasReq* pReq, const DasJdo* pParam return true; } -static bool _loadExamples(DasSrc* pThis) +static bool _DasSrc_loadExamples(DasSrc* pThis) { - const DasJdo* pEx = _jget(pThis->pIface, "examples"); - const DasJdo* pProto = _jget(pThis->pDoc, "protocol"); + const DasJdo* pEx = _dassrc_jget(pThis->pIface, "examples"); + const DasJdo* pProto = _dassrc_jget(pThis->pDoc, "protocol"); bool bLegacyEx = false; - if(!_jisary(pEx)){ - pEx = _jget(pProto, "examples"); - bLegacyEx = _jisdict(pEx); + if(!_dassrc_jisary(pEx)){ + pEx = _dassrc_jget(pProto, "examples"); + bLegacyEx = _dassrc_jisdict(pEx); if(!bLegacyEx) return true; } - size_t uMax = _jlen(pEx); + size_t uMax = _dassrc_jlen(pEx); if(uMax == 0) return true; pThis->aExample = (DasReq**)calloc(uMax, sizeof(DasReq*)); if(pThis->aExample == NULL) return false; @@ -624,13 +624,13 @@ static bool _loadExamples(DasSrc* pThis) if(pReq == NULL) return false; pThis->aExample[pThis->uExample] = pReq; pThis->uExample += 1; - const char* sLabel = _jstr(pEl->value, "title"); - if(sLabel == NULL) sLabel = _jstr(pEl->value, "name"); + const char* sLabel = _dassrc_jstr(pEl->value, "title"); + if(sLabel == NULL) sLabel = _dassrc_jstr(pEl->value, "name"); if(sLabel == NULL) sLabel = pEl->name->string; DasReq_setLabel(pReq, sLabel); DasSrc_stamp(pThis, pReq); - const DasJdo* pParams = _jget(pEl->value, "http_params"); - if(_jisdict(pParams) && !_exampleFromParams(pThis, pReq, pParams)) return false; + const DasJdo* pParams = _dassrc_jget(pEl->value, "http_params"); + if(_dassrc_jisdict(pParams) && !_DasSrc_exampleFromParams(pThis, pReq, pParams)) return false; } return true; } @@ -640,14 +640,14 @@ static bool _loadExamples(DasSrc* pThis) if(pReq == NULL) return false; pThis->aExample[pThis->uExample] = pReq; pThis->uExample += 1; - const char* sLabel = _jstr(pEl->value, "label"); - if(sLabel == NULL) sLabel = _jstr(pEl->value, "title"); - if(sLabel == NULL) sLabel = _jstr(pEl->value, "name"); + const char* sLabel = _dassrc_jstr(pEl->value, "label"); + if(sLabel == NULL) sLabel = _dassrc_jstr(pEl->value, "title"); + if(sLabel == NULL) sLabel = _dassrc_jstr(pEl->value, "name"); DasReq_setLabel(pReq, sLabel); DasSrc_stamp(pThis, pReq); - const DasJdo* pSettings = _jget(pEl->value, "settings"); + const DasJdo* pSettings = _dassrc_jget(pEl->value, "settings"); for(const das_json_dict_el* pP = DasJdo_dictFirst(pSettings); pP; pP = pP->next){ - if(!_exPathToKey(pP->name->string, sKey, sizeof(sKey))){ + if(!_dassrc_exPathToKey(pP->name->string, sKey, sizeof(sKey))){ daslog_warn_v("Ignoring example setting '%s' in %s, the path makes no sense", pP->name->string, pThis->sUrl); continue; @@ -670,7 +670,7 @@ void del_DasSrc(DasSrc* pThis) for(size_t u = 0; u < pThis->uProp; ++u){ free(pThis->aProp[u].sKey); free(pThis->aProp[u].sValue); - _clearCtrl(pThis->aProp[u].pCtrl); + _DasSrcCtrl_clear(pThis->aProp[u].pCtrl); } free(pThis->aProp); free(pThis->aCtrl); @@ -686,13 +686,13 @@ DasSrc* new_DasSrc_json(const char* sDoc, size_t uLen, const char* sPathUri, con { const char* sWhere = sUrl ? sUrl : (sPathUri ? sPathUri : "(memory)"); DasJdo* pDoc = das_json_parse(sDoc, uLen); - if(!_jisdict(pDoc)){ + if(!_dassrc_jisdict(pDoc)){ daslog_error_v("Source document %s is not a JSON object", sWhere); free(pDoc); return NULL; } - const char* sType = _jstr(pDoc, "type"); - if(!_strieq(sType, "HttpStreamSrc")){ + const char* sType = _dassrc_jstr(pDoc, "type"); + if(!_dassrc_strieq(sType, "HttpStreamSrc")){ daslog_error_v("Node %s is of type %s, only HttpStreamSrc nodes describe data sources " "this library can query", sWhere, sType ? sType : "(none)"); free(pDoc); @@ -710,29 +710,29 @@ DasSrc* new_DasSrc_json(const char* sDoc, size_t uLen, const char* sPathUri, con char sUri[600] = {'\0'}; if((sPathUri != NULL) && (sPathUri[0] != '\0')) _das_uri_expand(sPathUri, sUri, sizeof(sUri)); - pThis->sPath = _strdupz(sUri); - pThis->sUrl = _strdupz(sUrl); + pThis->sPath = _dassrc_strdupz(sUri); + pThis->sUrl = _dassrc_strdupz(sUrl); if((pThis->sPath == NULL)||(pThis->sUrl == NULL)){ del_DasSrc(pThis); return NULL; } - pThis->pIface = _jget(pDoc, "interface"); - if(!_jisdict(pThis->pIface)) pThis->pIface = NULL; + pThis->pIface = _dassrc_jget(pDoc, "interface"); + if(!_dassrc_jisdict(pThis->pIface)) pThis->pIface = NULL; - pStream->pProto = _jget(pDoc, "protocol"); - if(!_jisdict(pStream->pProto)) pStream->pProto = NULL; - pStream->pParams = _jget(pStream->pProto, "httpParams"); - if(pStream->pParams == NULL) pStream->pParams = _jget(pStream->pProto, "http_params"); - pStream->pBaseUrls = _jget(pStream->pProto, "baseUrls"); - if(pStream->pBaseUrls == NULL) pStream->pBaseUrls = _jget(pStream->pProto, "base_urls"); - if(!_jisary(pStream->pBaseUrls)) pStream->pBaseUrls = NULL; - pThis->bLegacy = (_jget(pStream->pProto, "http_params") != NULL) || - (_jget(pThis->pIface, "coordinates") != NULL); + pStream->pProto = _dassrc_jget(pDoc, "protocol"); + if(!_dassrc_jisdict(pStream->pProto)) pStream->pProto = NULL; + pStream->pParams = _dassrc_jget(pStream->pProto, "httpParams"); + if(pStream->pParams == NULL) pStream->pParams = _dassrc_jget(pStream->pProto, "http_params"); + pStream->pBaseUrls = _dassrc_jget(pStream->pProto, "baseUrls"); + if(pStream->pBaseUrls == NULL) pStream->pBaseUrls = _dassrc_jget(pStream->pProto, "base_urls"); + if(!_dassrc_jisary(pStream->pBaseUrls)) pStream->pBaseUrls = NULL; + pThis->bLegacy = (_dassrc_jget(pStream->pProto, "http_params") != NULL) || + (_dassrc_jget(pThis->pIface, "coordinates") != NULL); /* Two passes over the same walk: count, allocate exactly that many, * fill. Groups, properties and controls are three blocks, so a source * is a handful of allocations however large its interface, which * matters on a heap that never shrinks, as in a browser build. */ _walk_t w = {0, 0, 0, NULL, pStream->pParams, false}; - _walkIface(&w, pThis->pIface); + _dassrc_walkIface(&w, pThis->pIface); pThis->aGroup = (DasSrcGroup*)calloc(w.uGroup ? w.uGroup : 1, sizeof(DasSrcGroup)); pThis->aProp = (DasSrcProp*)calloc(w.uProp ? w.uProp : 1, sizeof(DasSrcProp)); pThis->aCtrl = (DasSrcCtrl*)calloc(w.uCtrl ? w.uCtrl : 1, sizeof(DasSrcCtrl)); @@ -744,7 +744,7 @@ DasSrc* new_DasSrc_json(const char* sDoc, size_t uLen, const char* sPathUri, con pThis->uCtrl = uCtrls; /* the fill pass checks against it */ w.uGroup = 0; w.uProp = 0; w.uCtrl = 0; w.pSrc = pThis; - _walkIface(&w, pThis->pIface); + _dassrc_walkIface(&w, pThis->pIface); pThis->uGroup = w.uGroup; pThis->uProp = w.uProp; pThis->uCtrl = w.uCtrl; @@ -754,7 +754,7 @@ DasSrc* new_DasSrc_json(const char* sDoc, size_t uLen, const char* sPathUri, con return NULL; } - if(!_loadExamples(pThis)){ + if(!_DasSrc_loadExamples(pThis)){ daslog_error_v("Couldn't read the examples of source %s", sWhere); del_DasSrc(pThis); return NULL; @@ -772,8 +772,8 @@ DasSrc* new_DasSrc_node(const DasNode* pNode) size_t uKids = dassrc_loadable(pNode, aNames, 8); char sMsg[512]; size_t n = snprintf(sMsg, sizeof(sMsg), "Node %s is not a data source", DasNode_srcUrl(pNode)); - if(uKids > 0) n = _catf(sMsg, sizeof(sMsg), n, ", its loadable children are:"); - for(size_t u = 0; (u < uKids) && (u < 8); ++u) n = _catf(sMsg, sizeof(sMsg), n, " %s", aNames[u]); + if(uKids > 0) n = _dassrc_catf(sMsg, sizeof(sMsg), n, ", its loadable children are:"); + for(size_t u = 0; (u < uKids) && (u < 8); ++u) n = _dassrc_catf(sMsg, sizeof(sMsg), n, " %s", aNames[u]); daslog_error_v("%s", sMsg); return NULL; } @@ -801,46 +801,46 @@ das_src_kind_e DasSrc_kind(const DasSrc* pThis){ return pThis->kind; } const char* DasSrc_path(const DasSrc* pThis){ return pThis->sPath; } const char* DasSrc_url(const DasSrc* pThis){ return pThis->sUrl; } -static const char* _docStr(const DasSrc* pThis, const char* sPath) +static const char* _DasSrc_docStr(const DasSrc* pThis, const char* sPath) { - const char* s = _jstr(pThis->pDoc, sPath); + const char* s = _dassrc_jstr(pThis->pDoc, sPath); return s ? s : ""; } const char* DasSrc_name(const DasSrc* pThis) { - const char* s = _jstr(pThis->pDoc, "label"); - if(s == NULL) s = _jstr(pThis->pDoc, "name"); + const char* s = _dassrc_jstr(pThis->pDoc, "label"); + if(s == NULL) s = _dassrc_jstr(pThis->pDoc, "name"); return s ? s : ""; } -const char* DasSrc_title(const DasSrc* pThis){ return _docStr(pThis, "title"); } -const char* DasSrc_version(const DasSrc* pThis){ return _docStr(pThis, "version"); } +const char* DasSrc_title(const DasSrc* pThis){ return _DasSrc_docStr(pThis, "title"); } +const char* DasSrc_version(const DasSrc* pThis){ return _DasSrc_docStr(pThis, "version"); } bool DasSrc_needsAuth(const DasSrc* pThis) { - const DasJdo* pProto = _jget(pThis->pDoc, "protocol"); - const DasJdo* pAuth = _jget(pProto, "authorization"); - if(pAuth == NULL) pAuth = _jget(pProto, "authentication"); - return _jtrue(_jget(pAuth, "required")); + const DasJdo* pProto = _dassrc_jget(pThis->pDoc, "protocol"); + const DasJdo* pAuth = _dassrc_jget(pProto, "authorization"); + if(pAuth == NULL) pAuth = _dassrc_jget(pProto, "authentication"); + return _dassrc_jtrue(_dassrc_jget(pAuth, "required")); } -static const DasJdo* _contacts(const DasSrc* pThis) +static const DasJdo* _DasSrc_contacts(const DasSrc* pThis) { - const DasJdo* p = _jget(pThis->pDoc, "contacts"); - if(!_jisary(p)) p = _jget(pThis->pDoc, "tech_contacts"); - return _jisary(p) ? p : NULL; + const DasJdo* p = _dassrc_jget(pThis->pDoc, "contacts"); + if(!_dassrc_jisary(p)) p = _dassrc_jget(pThis->pDoc, "tech_contacts"); + return _dassrc_jisary(p) ? p : NULL; } -size_t DasSrc_contactCount(const DasSrc* pThis){ return _jlen(_contacts(pThis)); } +size_t DasSrc_contactCount(const DasSrc* pThis){ return _dassrc_jlen(_DasSrc_contacts(pThis)); } const char* DasSrc_contact(const DasSrc* pThis, size_t uIdx, const char** psEmail) { - const das_json_ary_el* pEl = DasJdo_aryFirst(_contacts(pThis)); + const das_json_ary_el* pEl = DasJdo_aryFirst(_DasSrc_contacts(pThis)); for(size_t u = 0; (u < uIdx) && pEl; ++u) pEl = pEl->next; if(pEl == NULL) return NULL; - const char* sName = _jstr(pEl->value, "name"); - const char* sEmail = _jstr(pEl->value, "email"); + const char* sName = _dassrc_jstr(pEl->value, "name"); + const char* sEmail = _dassrc_jstr(pEl->value, "email"); if(psEmail) *psEmail = sEmail ? sEmail : ""; return sName ? sName : ""; } @@ -870,7 +870,7 @@ const DasSrcGroup* DasSrc_findGroup(const DasSrc* pThis, das_src_sec_e sec, cons if(sName == NULL) sName = ""; for(size_t u = 0; u < DasSrc_groupCount(pThis, sec); ++u){ const DasSrcGroup* pG = DasSrc_group(pThis, sec, u); - if(_strieq(pG->sName, sName)) return pG; + if(_dassrc_strieq(pG->sName, sName)) return pG; } return NULL; } @@ -880,14 +880,14 @@ const char* DasSrcGroup_name(const DasSrcGroup* pThis){ return pThis->sName; } const char* DasSrcGroup_label(const DasSrcGroup* pThis) { - const char* s = _jstr(pThis->pJdo, "label"); - if(s == NULL) s = _jstr(pThis->pJdo, "name"); + const char* s = _dassrc_jstr(pThis->pJdo, "label"); + if(s == NULL) s = _dassrc_jstr(pThis->pJdo, "name"); return s ? s : pThis->sName; } const char* DasSrcGroup_title(const DasSrcGroup* pThis) { - const char* s = _jstr(pThis->pJdo, "title"); + const char* s = _dassrc_jstr(pThis->pJdo, "title"); return s ? s : ""; } @@ -952,13 +952,13 @@ const DasSrcCtrl* DasSrcProp_ctrl(const DasSrcProp* pThis){ return pThis->pCtrl; * level. */ const char* DasSrcProp_units(const DasSrcProp* pThis) { - const DasJdo* pU = _jget(pThis->pJdo, "units"); + const DasJdo* pU = _dassrc_jget(pThis->pJdo, "units"); if(pU != NULL){ if(pU->type == das_json_type_str) return DasJdo_string(pU); - if(_jstr(pU, "value")) return _jstr(pU, "value"); + if(_dassrc_jstr(pU, "value")) return _dassrc_jstr(pU, "value"); } if(pThis->sec == das_src_sec_coord){ - const char* s = _jstr(_groupProps(pThis->pGroup->pJdo), "units/value"); + const char* s = _dassrc_jstr(_dassrc_groupProps(pThis->pGroup->pJdo), "units/value"); if(s != NULL) return s; } if(pThis->type == das_srcprop_time) return "UTC"; @@ -967,10 +967,10 @@ const char* DasSrcProp_units(const DasSrcProp* pThis) const char* DasSrcProp_title(const DasSrcProp* pThis, char* sBuf, size_t uLen) { - const char* s = _jstr(pThis->pJdo, "title"); - if(s == NULL) s = _jstr(pThis->pJdo, "summary"); - if(s == NULL) s = _jstr(pThis->pJdo, "label"); - if(s == NULL) s = _jstr(pThis->pJdo, "name"); + const char* s = _dassrc_jstr(pThis->pJdo, "title"); + if(s == NULL) s = _dassrc_jstr(pThis->pJdo, "summary"); + if(s == NULL) s = _dassrc_jstr(pThis->pJdo, "label"); + if(s == NULL) s = _dassrc_jstr(pThis->pJdo, "name"); if(s != NULL) return s; if(pThis->sec == das_src_sec_option) return ""; @@ -983,7 +983,7 @@ const char* DasSrcProp_title(const DasSrcProp* pThis, char* sBuf, size_t uLen) else{ /* A variable's title is usually the physical quantity, which reads better after "Output" than its short label does */ - const char* sTitle = _jstr(pThis->pGroup->pJdo, "title"); + const char* sTitle = _dassrc_jstr(pThis->pGroup->pJdo, "title"); snprintf(sBuf, uLen, "Output %s", sTitle ? sTitle : sGroup); } return sBuf; @@ -1037,16 +1037,16 @@ DasErrCode DasSrcCtrl_check(const DasSrcCtrl* pThis, const char* sVal, char* sWh switch(pThis->pProp->type){ case das_srcprop_bool:{ bool b; - if(_parseBool(sVal, &b)) return DAS_OKAY; + if(_dassrc_parseBool(sVal, &b)) return DAS_OKAY; snprintf(sWhy, uWhy, "'%s' is not a boolean value for %s, use true or false", sVal, sKey); return DASERR_SRC; } case das_srcprop_enum:{ for(size_t u = 0; u < pThis->uEnum; ++u) - if(_strieq(pThis->aEnum[u], sVal)) return DAS_OKAY; + if(_dassrc_strieq(pThis->aEnum[u], sVal)) return DAS_OKAY; size_t u = snprintf(sWhy, uWhy, "'%s' is not an allowed value for %s, choose one of:", sVal, sKey); for(size_t v = 0; v < pThis->uEnum; ++v) - u = _catf(sWhy, uWhy, u, "%s %s", v ? "," : "", pThis->aEnum[v]); + u = _dassrc_catf(sWhy, uWhy, u, "%s %s", v ? "," : "", pThis->aEnum[v]); return DASERR_SRC; } case das_srcprop_int: @@ -1063,7 +1063,7 @@ DasErrCode DasSrcCtrl_check(const DasSrcCtrl* pThis, const char* sVal, char* sWh /* "now" is how nodes spell an open ended range and servers take it in requests too */ das_time dt; - if(_strieq(sVal, "now") || dt_parsetime(&dt, sVal)) return DAS_OKAY; + if(_dassrc_strieq(sVal, "now") || dt_parsetime(&dt, sVal)) return DAS_OKAY; snprintf(sWhy, uWhy, "'%s' is not a time value for %s, use ISO 8601 such as " "2026-01-15T12:00", sVal, sKey); return DASERR_SRC; @@ -1092,7 +1092,7 @@ int _DasSrc_match(const DasSrc* pThis, const char* sKey, const DasSrcProp** aOut if(!_das_src_secByName(aParts[0], &sec)) return 0; for(size_t u = 0; u < pThis->uProp; ++u){ p = pThis->aProp + u; - if((p->sec == sec) && _strieq(p->pGroup->sName, aParts[1]) && + if((p->sec == sec) && _dassrc_strieq(p->pGroup->sName, aParts[1]) && _das_src_propEq(p->sName, aParts[2])){ _ADD_MATCH } } return n; @@ -1106,14 +1106,14 @@ int _DasSrc_match(const DasSrc* pThis, const char* sKey, const DasSrcProp** aOut p = pThis->aProp + u; if(p->sec != sec) continue; if((sec == das_src_sec_option) && _das_src_propEq(p->sName, aParts[1])){ _ADD_MATCH } - else if((sec != das_src_sec_option) && _strieq(p->pGroup->sName, aParts[1]) && + else if((sec != das_src_sec_option) && _dassrc_strieq(p->pGroup->sName, aParts[1]) && _das_src_propEq(p->sName, "enabled")){ _ADD_MATCH } } if(n > 0) return n; } for(size_t u = 0; u < pThis->uProp; ++u){ p = pThis->aProp + u; - if(_strieq(p->pGroup->sName, aParts[0]) && _das_src_propEq(p->sName, aParts[1])){ _ADD_MATCH } + if(_dassrc_strieq(p->pGroup->sName, aParts[0]) && _das_src_propEq(p->sName, aParts[1])){ _ADD_MATCH } } return n; } @@ -1122,7 +1122,7 @@ int _DasSrc_match(const DasSrc* pThis, const char* sKey, const DasSrcProp** aOut for(size_t u = 0; u < pThis->uProp; ++u){ p = pThis->aProp + u; if((p->sec == das_src_sec_option) && _das_src_propEq(p->sName, aParts[0])){ _ADD_MATCH } - else if((p->sec != das_src_sec_option) && _strieq(p->pGroup->sName, aParts[0]) && + else if((p->sec != das_src_sec_option) && _dassrc_strieq(p->pGroup->sName, aParts[0]) && _das_src_propEq(p->sName, "enabled")){ _ADD_MATCH } } if(n > 0) return n; @@ -1153,11 +1153,11 @@ const DasSrcGroup* _DasSrc_coordGroup(const DasSrc* pThis, const char* sKey) #define MAX_MATCH 8 -static void _ambiguous(const char* sKey, const DasSrcProp** aMatch, int n, char* sWhy, size_t uWhy) +static void _dassrc_ambiguous(const char* sKey, const DasSrcProp** aMatch, int n, char* sWhy, size_t uWhy) { size_t u = snprintf(sWhy, uWhy, "Key '%s' is ambiguous, use one of:", sKey); for(int i = 0; (i < n) && (i < MAX_MATCH); ++i) - u = _catf(sWhy, uWhy, u, " %s", aMatch[i]->sKey); + u = _dassrc_catf(sWhy, uWhy, u, " %s", aMatch[i]->sKey); } const DasSrcProp* DasSrc_findProp(const DasSrc* pThis, const char* sKey, char* sWhy, size_t uWhy) @@ -1169,7 +1169,7 @@ const DasSrcProp* DasSrc_findProp(const DasSrc* pThis, const char* sKey, char* s int n = _DasSrc_match(pThis, sKey, aMatch, MAX_MATCH); if(n == 1) return aMatch[0]; if(n == 0) snprintf(sWhy, uWhy, "No property named '%s' in this source", sKey); - else _ambiguous(sKey, aMatch, n, sWhy, uWhy); + else _dassrc_ambiguous(sKey, aMatch, n, sWhy, uWhy); return NULL; } @@ -1212,16 +1212,16 @@ DasReq* DasSrc_newReq(const DasSrc* pThis, int nExample) /* Identity is the catalog URI alone. The URL is where one copy of the * node happened to come from; a mirror or a local mount of the same node * has a different one and must validate the same requisitions. */ -static bool _stampMatches(const DasSrc* pThis, const DasReq* pReq) +static bool _DasSrc_stampMatches(const DasSrc* pThis, const DasReq* pReq) { const char* sPath = DasReq_srcPath(pReq); if(sPath[0] == '\0') return true; /* unstamped */ - return _strieq(sPath, pThis->sPath); + return _dassrc_strieq(sPath, pThis->sPath); } /* One item of a coordinate range: find the control, check the value, * add the derived entry. Returns false with sWhy filled on any trouble. */ -static bool _rangeItem( +static bool _dassrc_rangeItem( const DasSrc* pThis, DasReq* pReq, const DasSrcGroup* pG, const char* sProp, const char* sVal, bool* aPresent, bool* aCovered, char* sWhy, size_t uWhy ){ @@ -1242,7 +1242,7 @@ static bool _rangeItem( } /* GROUP=MIN,MAX[,STEP] or GROUP=SPAN, see "Keys" in source.h */ -static bool _rangeEntry( +static bool _dassrc_rangeEntry( const DasSrc* pThis, DasReq* pReq, const DasSrcGroup* pG, const char* sVal, bool* aPresent, bool* aCovered, char* sWhy, size_t uWhy ){ @@ -1262,15 +1262,15 @@ static bool _rangeEntry( char sBeg[40], sEnd[40]; dt_isoc(&dtBeg, sBeg, sizeof(sBeg), 0); dt_isoc(&dtEnd, sEnd, sizeof(sEnd), 0); - return _rangeItem(pThis, pReq, pG, "min", sBeg, aPresent, aCovered, sWhy, uWhy) && - _rangeItem(pThis, pReq, pG, "max", sEnd, aPresent, aCovered, sWhy, uWhy); + return _dassrc_rangeItem(pThis, pReq, pG, "min", sBeg, aPresent, aCovered, sWhy, uWhy) && + _dassrc_rangeItem(pThis, pReq, pG, "max", sEnd, aPresent, aCovered, sWhy, uWhy); } const char* aProps[3] = {"min", "max", NULL}; aProps[2] = DasSrcGroup_findProp(pG, "res") ? "res" : "inter"; char sCopy[SCALAR_SZ]; - _strcpyz(sCopy, sVal, sizeof(sCopy)); + _dassrc_strcpyz(sCopy, sVal, sizeof(sCopy)); char* pTok = sCopy; for(int i = 0; i < 4; ++i){ char* pComma = strchr(pTok, ','); @@ -1280,7 +1280,7 @@ static bool _rangeEntry( return false; } if(pTok[0] != '\0') - if(!_rangeItem(pThis, pReq, pG, aProps[i], pTok, aPresent, aCovered, sWhy, uWhy)) + if(!_dassrc_rangeItem(pThis, pReq, pG, aProps[i], pTok, aPresent, aCovered, sWhy, uWhy)) return false; if(pComma == NULL) break; pTok = pComma + 1; @@ -1291,7 +1291,7 @@ static bool _rangeEntry( DasErrCode DasSrc_check(const DasSrc* pThis, DasReq* pReq, size_t* puBad) { if(puBad) *puBad = 0; - if(!_stampMatches(pThis, pReq)){ + if(!_DasSrc_stampMatches(pThis, pReq)){ daslog_error_v("Requisition is for source %s%s, not %s%s", DasReq_srcPath(pReq), DasReq_srcPath(pReq)[0] ? "" : DasReq_srcUrl(pReq), pThis->sPath, pThis->sPath[0] ? "" : pThis->sUrl); @@ -1316,7 +1316,7 @@ DasErrCode DasSrc_check(const DasSrc* pThis, DasReq* pReq, size_t* puBad) const DasSrcGroup* pG = _DasSrc_coordGroup(pThis, sKey); if(pG != NULL){ - if(_rangeEntry(pThis, pReq, pG, sVal, aPresent, aCovered, sWhy, sizeof(sWhy))) + if(_dassrc_rangeEntry(pThis, pReq, pG, sVal, aPresent, aCovered, sWhy, sizeof(sWhy))) _DasReq_setVerdict(pReq, u, das_req_ok, NULL); else{ _DasReq_setVerdict(pReq, u, das_req_bad, sWhy); @@ -1332,7 +1332,7 @@ DasErrCode DasSrc_check(const DasSrc* pThis, DasReq* pReq, size_t* puBad) _DasReq_setVerdict(pReq, u, das_req_bad, sWhy); ++uBad; continue; } if(n > 1){ - _ambiguous(sKey, aMatch, n, sWhy, sizeof(sWhy)); + _dassrc_ambiguous(sKey, aMatch, n, sWhy, sizeof(sWhy)); _DasReq_setVerdict(pReq, u, das_req_bad, sWhy); ++uBad; continue; } const DasSrcProp* p = aMatch[0]; @@ -1387,7 +1387,7 @@ DasErrCode DasSrc_check(const DasSrc* pThis, DasReq* pReq, size_t* puBad) } /* The format group a requisition turns on, else the node's default */ -static const DasSrcGroup* _chosenFormat(const DasSrc* pThis, const DasReq* pReq) +static const DasSrcGroup* _DasSrc_chosenFormat(const DasSrc* pThis, const DasReq* pReq) { const DasSrcGroup* pDef = NULL; for(size_t g = 0; g < DasSrc_groupCount(pThis, das_src_sec_format); ++g){ @@ -1395,13 +1395,13 @@ static const DasSrcGroup* _chosenFormat(const DasSrc* pThis, const DasReq* pReq) const DasSrcProp* pOn = DasSrcGroup_enable(pG); if(pOn == NULL) continue; bool b = false; - if((pDef == NULL) && _parseBool(pOn->sValue, &b) && b) pDef = pG; + if((pDef == NULL) && _dassrc_parseBool(pOn->sValue, &b) && b) pDef = pG; if(pReq == NULL) continue; for(size_t u = 0; u < pReq->uEntry; ++u){ if(pReq->aEntry[u].bDerived) continue; const DasSrcProp* aMatch[1]; if(_DasSrc_match(pThis, pReq->aEntry[u].sKey, aMatch, 1) != 1) continue; - if((aMatch[0] == pOn) && _parseBool(pReq->aEntry[u].sVal, &b) && b) return pG; + if((aMatch[0] == pOn) && _dassrc_parseBool(pReq->aEntry[u].sVal, &b) && b) return pG; } } return pDef; @@ -1409,33 +1409,33 @@ static const DasSrcGroup* _chosenFormat(const DasSrc* pThis, const DasReq* pReq) const char* DasSrc_mime(const DasSrc* pThis, const DasReq* pReq) { - const DasSrcGroup* pG = _chosenFormat(pThis, pReq); + const DasSrcGroup* pG = _DasSrc_chosenFormat(pThis, pReq); if((pG != NULL) && (pG->uMime > 0)) return pG->aMime[0]; /* 2018 nodes state a default format outside the interface */ - const char* s = _jstr(pThis->pDoc, "format/default/mime"); - if(s == NULL) s = _jstr(pThis->pDoc, "protocol/format/default/mime"); - if(s == NULL) s = _jstr(pThis->pDoc, "interface/format/default/mime"); + const char* s = _dassrc_jstr(pThis->pDoc, "format/default/mime"); + if(s == NULL) s = _dassrc_jstr(pThis->pDoc, "protocol/format/default/mime"); + if(s == NULL) s = _dassrc_jstr(pThis->pDoc, "interface/format/default/mime"); return s ? s : ""; } /* ************************************************************************* */ /* Stream sources: translating a requisition to protocol parameters */ -static const DasStreamSrc* _stream(const DasSrc* pThis) +static const DasStreamSrc* _DasSrc_stream(const DasSrc* pThis) { return (pThis->kind == das_src_kind_stream) ? (const DasStreamSrc*)pThis : NULL; } size_t DasStreamSrc_urlCount(const DasSrc* pThis) { - const DasStreamSrc* pS = _stream(pThis); - return pS ? _jlen(pS->pBaseUrls) : 0; + const DasStreamSrc* pS = _DasSrc_stream(pThis); + return pS ? _dassrc_jlen(pS->pBaseUrls) : 0; } const char* DasStreamSrc_baseUrl(const DasSrc* pThis, size_t uIdx) { - const DasStreamSrc* pS = _stream(pThis); + const DasStreamSrc* pS = _DasSrc_stream(pThis); if(pS == NULL) return NULL; const das_json_ary_el* pEl = DasJdo_aryFirst(pS->pBaseUrls); for(size_t u = 0; (u < uIdx) && pEl; ++u) pEl = pEl->next; @@ -1447,16 +1447,16 @@ const char* DasStreamSrc_baseUrl(const DasSrc* pThis, size_t uIdx) typedef struct { const char* sName; char* sVal; } _param_t; typedef struct { _param_t* a; size_t u; size_t uCap; } _out_t; -static _param_t* _outFind(_out_t* pO, const char* sName) +static _param_t* _dassrc_outFind(_out_t* pO, const char* sName) { for(size_t u = 0; u < pO->u; ++u) if(strcmp(pO->a[u].sName, sName) == 0) return pO->a + u; return NULL; } -static bool _outSet(_out_t* pO, const char* sName, const char* sVal) +static bool _dassrc_outSet(_out_t* pO, const char* sName, const char* sVal) { - _param_t* p = _outFind(pO, sName); + _param_t* p = _dassrc_outFind(pO, sName); if(p == NULL){ if(pO->u == pO->uCap){ size_t uCap = pO->uCap ? pO->uCap * 2 : 16; @@ -1470,17 +1470,17 @@ static bool _outSet(_out_t* pO, const char* sName, const char* sVal) p->sVal = NULL; pO->u += 1; } - char* sNew = _strdupz(sVal); + char* sNew = _dassrc_strdupz(sVal); if(sNew == NULL) return false; free(p->sVal); p->sVal = sNew; return true; } -static bool _outAddFlag(_out_t* pO, const char* sName, const char* sFlagVal, const char* sSep) +static bool _dassrc_outAddFlag(_out_t* pO, const char* sName, const char* sFlagVal, const char* sSep) { - _param_t* p = _outFind(pO, sName); - if((p == NULL)||(p->sVal[0] == '\0')) return _outSet(pO, sName, sFlagVal); + _param_t* p = _dassrc_outFind(pO, sName); + if((p == NULL)||(p->sVal[0] == '\0')) return _dassrc_outSet(pO, sName, sFlagVal); char* sNew = das_string("%s%s%s", p->sVal, sSep, sFlagVal); if(sNew == NULL) return false; free(p->sVal); @@ -1488,7 +1488,7 @@ static bool _outAddFlag(_out_t* pO, const char* sName, const char* sFlagVal, con return true; } -static void _outClear(_out_t* pO) +static void _dassrc_outClear(_out_t* pO) { for(size_t u = 0; u < pO->u; ++u) free(pO->a[u].sVal); free(pO->a); @@ -1507,34 +1507,34 @@ static void _outClear(_out_t* pO) * with the parameter's flagSep * - otherwise the value as given, or the block's pval */ -static bool _translate(const DasSrcCtrl* pC, const char* sVal, _out_t* pO, char* sWhy, size_t uWhy) +static bool _DasSrcCtrl_translate(const DasSrcCtrl* pC, const char* sVal, _out_t* pO, char* sWhy, size_t uWhy) { const DasSrcProp* p = pC->pProp; char sFlag[DAS_MAX_ID_BUFSZ] = {'\0'}; char sPVal[SCALAR_SZ] = {'\0'}; - _strcpyz(sFlag, _jstr(pC->pSet, "flag"), sizeof(sFlag)); - bool bHavePVal = _das_json_scalar(_jget(pC->pSet, "pval"), sPVal, sizeof(sPVal)); + _dassrc_strcpyz(sFlag, _dassrc_jstr(pC->pSet, "flag"), sizeof(sFlag)); + bool bHavePVal = _das_json_scalar(_dassrc_jget(pC->pSet, "pval"), sPVal, sizeof(sPVal)); - const DasJdo* pEnum = _jget(pC->pSet, "enum"); - if(_jisary(pEnum)){ + const DasJdo* pEnum = _dassrc_jget(pC->pSet, "enum"); + if(_dassrc_jisary(pEnum)){ char sItem[SCALAR_SZ]; for(const das_json_ary_el* pEl = DasJdo_aryFirst(pEnum); pEl; pEl = pEl->next){ - const DasJdo* pItem = _jisdict(pEl->value) ? _jget(pEl->value, "value") : pEl->value; + const DasJdo* pItem = _dassrc_jisdict(pEl->value) ? _dassrc_jget(pEl->value, "value") : pEl->value; if(!_das_json_scalar(pItem, sItem, sizeof(sItem))) continue; - if(!_strieq(sItem, sVal)) continue; - if(_das_json_scalar(_jget(pEl->value, "pval"), sPVal, sizeof(sPVal))) bHavePVal = true; - else if(!bHavePVal){ _strcpyz(sPVal, sItem, sizeof(sPVal)); bHavePVal = true; } - if(_jstr(pEl->value, "flag")) _strcpyz(sFlag, _jstr(pEl->value, "flag"), sizeof(sFlag)); + if(!_dassrc_strieq(sItem, sVal)) continue; + if(_das_json_scalar(_dassrc_jget(pEl->value, "pval"), sPVal, sizeof(sPVal))) bHavePVal = true; + else if(!bHavePVal){ _dassrc_strcpyz(sPVal, sItem, sizeof(sPVal)); bHavePVal = true; } + if(_dassrc_jstr(pEl->value, "flag")) _dassrc_strcpyz(sFlag, _dassrc_jstr(pEl->value, "flag"), sizeof(sFlag)); break; } } else if(p->type == das_srcprop_bool){ bool bUser = false; - _parseBool(sVal, &bUser); - const DasJdo* pTrig = _jget(pC->pSet, "value"); - if(_jisbool(pTrig) && (bUser != _jtrue(pTrig))) return true; + _dassrc_parseBool(sVal, &bUser); + const DasJdo* pTrig = _dassrc_jget(pC->pSet, "value"); + if(_dassrc_jisbool(pTrig) && (bUser != _dassrc_jtrue(pTrig))) return true; if(!bHavePVal && (sFlag[0] == '\0')){ - _strcpyz(sPVal, bUser ? "true" : "false", sizeof(sPVal)); + _dassrc_strcpyz(sPVal, bUser ? "true" : "false", sizeof(sPVal)); bHavePVal = true; } } @@ -1542,29 +1542,29 @@ static bool _translate(const DasSrcCtrl* pC, const char* sVal, _out_t* pO, char* if(sFlag[0] != '\0'){ char sPath[DAS_MAX_ID_BUFSZ + 8]; snprintf(sPath, sizeof(sPath), "flags/%s", sFlag); - const DasJdo* pF = _jget(pC->pParam, sPath); + const DasJdo* pF = _dassrc_jget(pC->pParam, sPath); if(pF == NULL){ snprintf(sWhy, uWhy, "Flag '%s' for %s is not defined under protocol parameter %s, " "the node is broken", sFlag, p->sKey, pC->sParam); return false; } char sFVal[SCALAR_SZ], sFull[SCALAR_SZ]; - if(!_das_json_scalar(_jget(pF, "value"), sFVal, sizeof(sFVal))) _strcpyz(sFVal, sFlag, sizeof(sFVal)); - const char* sPrefix = _jstr(pF, "prefix"); + if(!_das_json_scalar(_dassrc_jget(pF, "value"), sFVal, sizeof(sFVal))) _dassrc_strcpyz(sFVal, sFlag, sizeof(sFVal)); + const char* sPrefix = _dassrc_jstr(pF, "prefix"); snprintf(sFull, sizeof(sFull), "%s%s", sPrefix ? sPrefix : "", sFVal); - const char* sSep = _jstr(pC->pParam, "flagSep"); - if(sSep == NULL) sSep = _jstr(pC->pParam, "flag_sep"); + const char* sSep = _dassrc_jstr(pC->pParam, "flagSep"); + if(sSep == NULL) sSep = _dassrc_jstr(pC->pParam, "flag_sep"); if(sSep == NULL) sSep = " "; - if(!_outAddFlag(pO, pC->sParam, sFull, sSep)){ snprintf(sWhy, uWhy, "Out of memory"); return false; } + if(!_dassrc_outAddFlag(pO, pC->sParam, sFull, sSep)){ snprintf(sWhy, uWhy, "Out of memory"); return false; } return true; } - if(!_outSet(pO, pC->sParam, bHavePVal ? sPVal : sVal)){ snprintf(sWhy, uWhy, "Out of memory"); return false; } + if(!_dassrc_outSet(pO, pC->sParam, bHavePVal ? sPVal : sVal)){ snprintf(sWhy, uWhy, "Out of memory"); return false; } return true; } /* The value a requisition gives a property: the last good non-empty * entry that resolved to it, derived or not */ -static const char* _reqValueFor(const DasReq* pReq, size_t uProp) +static const char* _dassrc_reqValueFor(const DasReq* pReq, size_t uProp) { const char* sVal = NULL; for(size_t u = 0; u < pReq->uEntry; ++u){ @@ -1575,20 +1575,20 @@ static const char* _reqValueFor(const DasReq* pReq, size_t uProp) return sVal; } -static bool _sameAsDefault(const DasSrcProp* p, const char* sVal) +static bool _DasSrcProp_sameAsDefault(const DasSrcProp* p, const char* sVal) { if(p->sValue == NULL) return false; if(p->type == das_srcprop_bool){ bool bA = false, bB = false; - return _parseBool(sVal, &bA) && _parseBool(p->sValue, &bB) && (bA == bB); + return _dassrc_parseBool(sVal, &bA) && _dassrc_parseBool(p->sValue, &bB) && (bA == bB); } - return _strieq(sVal, p->sValue); + return _dassrc_strieq(sVal, p->sValue); } size_t DasStreamSrc_url(const DasSrc* pThis, DasReq* pReq, size_t uBase, char* sBuf, size_t uLen) { if(uLen > 0) sBuf[0] = '\0'; - const DasStreamSrc* pS = _stream(pThis); + const DasStreamSrc* pS = _DasSrc_stream(pThis); if((pS == NULL)||(pS->pProto == NULL)||(pS->pBaseUrls == NULL)){ daslog_error_v("Source %s has no protocol section with base URLs, it cannot be queried", pThis->sUrl); @@ -1618,7 +1618,7 @@ size_t DasStreamSrc_url(const DasSrc* pThis, DasReq* pReq, size_t uBase, char* s const DasSrcCtrl* pC = p->pCtrl; if(pC == NULL) continue; - const char* sVal = _reqValueFor(pReq, u); + const char* sVal = _dassrc_reqValueFor(pReq, u); bool bDefault = false; if(sVal == NULL){ if(!pC->bRequired || (p->sValue == NULL)) continue; @@ -1627,58 +1627,58 @@ size_t DasStreamSrc_url(const DasSrc* pThis, DasReq* pReq, size_t uBase, char* s } /* A required default fills a hole, it never replaces what another control already put in a shared parameter */ - if(bDefault && _outFind(&out, pC->sParam)) continue; + if(bDefault && _dassrc_outFind(&out, pC->sParam)) continue; /* A value equal to the default carries no information unless the server insists on hearing it */ - if(!bDefault && !pC->bRequired && _sameAsDefault(p, sVal)) continue; + if(!bDefault && !pC->bRequired && _DasSrcProp_sameAsDefault(p, sVal)) continue; - if(!_translate(pC, sVal, &out, sWhy, sizeof(sWhy))){ + if(!_DasSrcCtrl_translate(pC, sVal, &out, sWhy, sizeof(sWhy))){ daslog_error_v("%s", sWhy); - _outClear(&out); + _dassrc_outClear(&out); return 0; } } for(size_t u = 0; u < pReq->uRaw; ++u){ - if(!_outSet(&out, pReq->aRaw[u].sName, pReq->aRaw[u].sVal)){ _outClear(&out); return 0; } + if(!_dassrc_outSet(&out, pReq->aRaw[u].sName, pReq->aRaw[u].sVal)){ _dassrc_outClear(&out); return 0; } } /* Protocol parameters the node marks required, whatever supplies them */ for(const das_json_dict_el* pEl = DasJdo_dictFirst(pS->pParams); pEl; pEl = pEl->next){ - if(!_jtrue(_jget(pEl->value, "required"))) continue; + if(!_dassrc_jtrue(_dassrc_jget(pEl->value, "required"))) continue; const char* sName = pEl->name->string; - if(_outFind(&out, sName) != NULL) continue; + if(_dassrc_outFind(&out, sName) != NULL) continue; size_t n = snprintf(sWhy, sizeof(sWhy), "Required parameter %s missing, set it with", sName); int nKeys = 0; for(size_t i = 0; i < pThis->uProp; ++i){ const DasSrcCtrl* pC = pThis->aProp[i].pCtrl; if(pC && (strcmp(pC->sParam, sName) == 0)){ - n = _catf(sWhy, sizeof(sWhy), n, "%s %s=VALUE", nKeys ? " or" : "", pThis->aProp[i].sKey); + n = _dassrc_catf(sWhy, sizeof(sWhy), n, "%s %s=VALUE", nKeys ? " or" : "", pThis->aProp[i].sKey); ++nKeys; } } - if(nKeys == 0) _catf(sWhy, sizeof(sWhy), n, " a raw parameter, no interface key sets it"); + if(nKeys == 0) _dassrc_catf(sWhy, sizeof(sWhy), n, " a raw parameter, no interface key sets it"); daslog_error_v("%s", sWhy); - _outClear(&out); + _dassrc_outClear(&out); return 0; } - size_t u = _catf(sBuf, uLen, 0, "%s", sBase); + size_t u = _dassrc_catf(sBuf, uLen, 0, "%s", sBase); size_t uNeed = strlen(sBase); const char* sJoin = strchr(sBase, '?') ? "&" : "?"; for(size_t i = 0; i < out.u; ++i){ size_t uEnc = das_url_encode(out.a[i].sVal, NULL, 0) + 1; char* sEnc = (char*)malloc(uEnc); - if(sEnc == NULL){ _outClear(&out); return 0; } + if(sEnc == NULL){ _dassrc_outClear(&out); return 0; } das_url_encode(out.a[i].sVal, sEnc, uEnc); - u = _catf(sBuf, uLen, u, "%s%s=%s", sJoin, out.a[i].sName, sEnc); + u = _dassrc_catf(sBuf, uLen, u, "%s%s=%s", sJoin, out.a[i].sName, sEnc); uNeed += strlen(sJoin) + strlen(out.a[i].sName) + 1 + strlen(sEnc); free(sEnc); sJoin = "&"; } - _outClear(&out); + _dassrc_outClear(&out); return uNeed; } From ca84226f2bf93a9f01b557bd8e4cfda3073135f7 Mon Sep 17 00:00:00 2001 From: cpiker Date: Wed, 28 Oct 2026 04:58:29 -0500 Subject: [PATCH 16/18] correcting include files for windows --- utilities/das_fetch.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utilities/das_fetch.c b/utilities/das_fetch.c index c05749e..7720cde 100644 --- a/utilities/das_fetch.c +++ b/utilities/das_fetch.c @@ -35,7 +35,6 @@ #include #include #include -#include #include #include #include @@ -54,9 +53,11 @@ #include #define isatty _isatty #define fileno _fileno +#define strncasecmp _strnicmp /* MSVC keeps the POSIX one under its own name */ #else #include #include +#include /* strncasecmp */ #endif #define PROG "das_fetch" From b012b7e52e182103110c93c754beceb81821d577 Mon Sep 17 00:00:00 2001 From: "C. Piker" Date: Wed, 28 Oct 2026 12:11:22 -0500 Subject: [PATCH 17/18] util: Windows dir sep fix --- das3/util.c | 10 ++++++---- test/TestCat.c | 3 +++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/das3/util.c b/das3/util.c index 6a7de3c..abd41de 100644 --- a/das3/util.c +++ b/das3/util.c @@ -818,7 +818,8 @@ bool das_copyfile(const char* src, const char* dest) uLen = strlen(sPath); for(u = 0; u0) && (u < uLen-1)){ + if(((sPath[u] == '/')||(sPath[u] == DAS_DSEPC)) && (u>0) && (u < uLen-1)){ + char cSep = sPath[u]; sPath[u] = '\0'; if(! das_isdir(sPath) ){ @@ -838,7 +839,7 @@ bool das_copyfile(const char* src, const char* dest) } } - sPath[u] = '/'; + sPath[u] = cSep; } } @@ -889,7 +890,8 @@ DasErrCode das_mkdirsto(const char* path) int nErr; for(size_t u = 0; u < uLen; u++){ - if((sPath[u] == DAS_DSEPC) && (u>0) && (u < uLen-1)){ + if(((sPath[u] == '/')||(sPath[u] == DAS_DSEPC)) && (u>0) && (u < uLen-1)){ + char cSep = sPath[u]; sPath[u] = '\0'; if(! das_isdir(sPath) ){ @@ -907,7 +909,7 @@ DasErrCode das_mkdirsto(const char* path) } } - sPath[u] = '/'; + sPath[u] = cSep; } } return DAS_OKAY; diff --git a/test/TestCat.c b/test/TestCat.c index 90d82cd..c86558b 100644 --- a/test/TestCat.c +++ b/test/TestCat.c @@ -37,7 +37,10 @@ #include #include #include + +#ifndef _WIN32 #include +#endif #include #include From ae4ed920d794ab508abdc1a1e48ad57e2aa9f9cd Mon Sep 17 00:00:00 2001 From: "C. Piker" Date: Wed, 28 Oct 2026 13:20:37 -0500 Subject: [PATCH 18/18] windows: Build fixes, directory handling - das_mkdirsto() now handlesd '/' and '\' it also skips the leading leading drive spec (ex "C:") - Cast const char**, and wrap pragmas to quiet MSC builds Developed-with: Claude Opus 5.5 --- das3/catalog.c | 6 +++--- das3/source.c | 2 +- das3/util.c | 21 ++++++++++++++++++--- test/TestVar.c | 2 +- utilities/das2_cache_rdr.c | 8 ++++++++ utilities/das2_from_cdf.c | 2 +- utilities/das3_from_cdf.c | 2 +- 7 files changed, 33 insertions(+), 10 deletions(-) diff --git a/das3/catalog.c b/das3/catalog.c index eb36d73..cbb49c5 100644 --- a/das3/catalog.c +++ b/das3/catalog.c @@ -788,7 +788,7 @@ static DasNode* _DasCat_table(DasCat* pThis) const char** aUri = (const char**)calloc(pThis->uMounts, sizeof(char*)); if((aUrl == NULL)||(aUri == NULL)){ das_error(DASERR_CAT, "Couldn't allocate the mount table"); - free(aUrl); free(aUri); + free((void*)aUrl); free((void*)aUri); /* quiet msvc */ return NULL; } for(size_t u = 0; u < pThis->uMounts; ++u){ @@ -807,8 +807,8 @@ static DasNode* _DasCat_table(DasCat* pThis) } } pThis->pTable = _dasnode_newMountTable(aUrl, aUri, pThis->uMounts); - free(aUrl); - free(aUri); + free((void*)aUrl); /* quiet msvc */ + free((void*)aUri); /* quiet msvc */ return _DasCat_keep(pThis, pThis->pTable); } diff --git a/das3/source.c b/das3/source.c index 229aa75..c0cb21c 100644 --- a/das3/source.c +++ b/das3/source.c @@ -674,7 +674,7 @@ void del_DasSrc(DasSrc* pThis) } free(pThis->aProp); free(pThis->aCtrl); - for(size_t u = 0; u < pThis->uGroup; ++u) free(pThis->aGroup[u].aMime); + for(size_t u = 0; u < pThis->uGroup; ++u) free((void*)pThis->aGroup[u].aMime); /* quiet msvc */ free(pThis->aGroup); free(pThis->pDoc); free(pThis->sPath); diff --git a/das3/util.c b/das3/util.c index abd41de..62bc291 100644 --- a/das3/util.c +++ b/das3/util.c @@ -775,6 +775,19 @@ void das_stdio_binary(FILE* pFile) #endif } +/* Length of a leading drive spec, such as "C:", that's never made as a + directory. stat("C:") doesn't report a directory, so mkdir would be + tried and fail. */ +static size_t _das_drivelen(const char* sPath) +{ +#ifdef WIN32 + if((sPath[0] != '\0') && (sPath[1] == ':')) return 2; +#else + (void)sPath; +#endif + return 0; +} + #ifndef WIN32 bool das_copyfile(const char* src, const char* dest, mode_t mode) #else @@ -816,9 +829,10 @@ bool das_copyfile(const char* src, const char* dest) strncpy(sPath, dest, 255); uLen = strlen(sPath); - + size_t uDrive = _das_drivelen(sPath); + for(u = 0; u0) && (u < uLen-1)){ + if(((sPath[u] == '/')||(sPath[u] == DAS_DSEPC)) && (u>uDrive) && (u < uLen-1)){ char cSep = sPath[u]; sPath[u] = '\0'; @@ -887,10 +901,11 @@ DasErrCode das_mkdirsto(const char* path) #endif size_t uLen = strlen(sPath); + size_t uDrive = _das_drivelen(sPath); int nErr; for(size_t u = 0; u < uLen; u++){ - if(((sPath[u] == '/')||(sPath[u] == DAS_DSEPC)) && (u>0) && (u < uLen-1)){ + if(((sPath[u] == '/')||(sPath[u] == DAS_DSEPC)) && (u>uDrive) && (u < uLen-1)){ char cSep = sPath[u]; sPath[u] = '\0'; diff --git a/test/TestVar.c b/test/TestVar.c index 8842ef2..42dc7eb 100644 --- a/test/TestVar.c +++ b/test/TestVar.c @@ -288,7 +288,7 @@ static int test_byte_runs(void) MUST(DasVar_get((DasVar*)pStr, aLoc, DAS_BS_NULL, &dm) == 0); /* sVal is read below */ CHECK(dm.vt == vtText); const char* sVal = NULL; - memcpy(&sVal, &dm, sizeof(const char*)); + memcpy((void*)&sVal, &dm, sizeof(const char*)); /* quiet msvc */ MUST(sVal != NULL); CHECK(strcmp(sVal, "BURST") == 0); diff --git a/utilities/das2_cache_rdr.c b/utilities/das2_cache_rdr.c index 9ca3183..e793d50 100644 --- a/utilities/das2_cache_rdr.c +++ b/utilities/das2_cache_rdr.c @@ -445,8 +445,10 @@ char** timeBinFileList( /* Getting generic binned parameter lists */ /* Get GCC to shut it's cakehole about the unused parameters below */ +#ifdef __GNUC__ /* quiet msvc */ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" +#endif char** generalBinFileList( const char* sRoot, cache_tree_t* pTree, double rBeg, double rEnd, @@ -457,7 +459,9 @@ char** generalBinFileList( return NULL; } +#ifdef __GNUC__ #pragma GCC diagnostic pop +#endif /* ************************************************************************* */ /* Writing output data */ @@ -560,13 +564,17 @@ DasErrCode onPktData(PktDesc* pPktIn, void* vpHDat){ /* Callback to ignore no data messages, we'll count and make our own */ /* Like I said. GCC, just freaking chill okay */ +#ifdef __GNUC__ /* quiet msvc */ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" +#endif DasErrCode onIgnoreNoData(OobExcept* pExcept, void* vpHDat) { return DAS_OKAY; } +#ifdef __GNUC__ #pragma GCC diagnostic pop +#endif /* ************************************************************************* */ int readCache( diff --git a/utilities/das2_from_cdf.c b/utilities/das2_from_cdf.c index ed6e5d1..2810543 100644 --- a/utilities/das2_from_cdf.c +++ b/utilities/das2_from_cdf.c @@ -395,7 +395,7 @@ int parseArgs(int argc, char** argv, popts_t* pOpts) strncpy(pOpts->range.sCoord, "time", sizeof(pOpts->range.sCoord) - 1); int nLater = pOpts->nVars; const char* asLater[MAX_DATA_VARS]; - memcpy(asLater, pOpts->asVars, nLater * sizeof(const char*)); + memcpy((void*)asLater, pOpts->asVars, nLater * sizeof(const char*)); /* quiet msvc */ pOpts->nVars = 0; if((sBeg != NULL)&&(_addVars(pOpts, sBeg) != DAS_OKAY)) return PERR; if((sEnd != NULL)&&(_addVars(pOpts, sEnd) != DAS_OKAY)) return PERR; diff --git a/utilities/das3_from_cdf.c b/utilities/das3_from_cdf.c index b4da0d2..d9ad5fe 100644 --- a/utilities/das3_from_cdf.c +++ b/utilities/das3_from_cdf.c @@ -548,7 +548,7 @@ int parseArgs(int argc, char** argv, popts_t* pOpts) and they came before the ones already added */ int nLater = pOpts->nVars; const char* asLater[MAX_DATA_VARS]; - memcpy(asLater, pOpts->asVars, nLater * sizeof(const char*)); + memcpy((void*)asLater, pOpts->asVars, nLater * sizeof(const char*)); /* quiet msvc */ pOpts->nVars = 0; if((sBeg != NULL)&&(_addVars(pOpts, sBeg) != DAS_OKAY)) return PERR; if((sEnd != NULL)&&(_addVars(pOpts, sEnd) != DAS_OKAY)) return PERR;