* New yang changelog experimental feature for automatic upgrade

* Added modules-state diff parameter to xmldb_get datastore function for startup scenarios.
* Allowed Yang extended Xpath functions (syntax only):
  * re-match, deref, derived-from, derived-from-or-self, enum-value, bit-is-set
* XSD regular expression handling of dash(`-`)
  *: Translate XDS `[xxx\-yyy]` to POSIX `[xxxyyy-]`.
* YANG Anydata treated same as Anyxml

CHANGELOG.md

@@ -1,6 +1,6 @@
 # Clixon Changelog
 
-## 3.10.0 (Upcoming)
+## 3.10.0/4.0.0 (Upcoming)
 
 ### Major New features
 * Persistent CLI history: [Preserve CLI command history across sessions. The up/down arrows](https://github.com/clicon/clixon/issues/79)
@@ -18,16 +18,22 @@
     * Check which modules match, and which do not.
   * Loading of "extra" XML.
   * Detection of in-compatible XML and Yang models in the startup configuration.
-  * An upgrade callback when in-compatible XML is encountered (`ca_upgrade`)
+  * A user can register upgrade callbacks per module/revision when in-compatible XML is encountered (`update_callback_register`).
   * A "failsafe" mode allowing a user to repair the startup on errors or failed validation.
   * Major rewrite of `backend_main.c` and a new module `backend_startup.c`
+* New yang changelog experimental feature for automatic upgrade
+  * Yang module [email protected] based on draft-wang-netmod-module-revision-management-01
+  * Two config options control:
+    * CLICON_YANG_CHANGELOG enables the yang changelog feature
+    * CLICON_YANG_CHANGELOG_FILE where the changelog resides
 * Datastore files contain RFC7895 module-state information
-  * Added modules-state parameter to xmldb_get datastore function
+  * Added modules-state diff parameter to xmldb_get datastore function
   * Set config option `CLICON_XMLDB_MODSTATE` to true
     * Enable this if you wish to use the upgrade feature in the new startup functionality.
   * Note that this adds bytes to your configs
 
 ### API changes on existing features (you may need to change your code)
+* Added modules-state diff parameter to xmldb_get datastore function for startup scenarios. Set this to NULL in normal cases.
 * `rpc_callback_register` added a namespace parameter. Example:
    ```
      rpc_callback_register(h, empty_rpc, NULL, "urn:example:clixon", "empty");
@@ -62,6 +68,11 @@
 * Added libgen.h for baseline()
 
 ### Corrected Bugs
+* Allowed Yang extended Xpath functions (syntax only):
+  * re-match, deref, derived-from, derived-from-or-self, enum-value, bit-is-set
+* XSD regular expression handling of dash(`-`)
+  *: Translate XDS `[xxx\-yyy]` to POSIX `[xxxyyy-]`.
+* YANG Anydata treated same as Anyxml 
 * Bugfix: [Nodes from more than one of the choice's branches exist at the same time](https://github.com/clicon/clixon/issues/81)
   * Note it may still be possible to load a file with multiple choice elements via netconf, but it will not pass validate.
 * Bugfix: Default NACM policies applied even if NACM is disabled

README.md

@@ -113,6 +113,7 @@ However, the following YANG syntax modules are not implemented:
 - unique
 - action
 - refine
+- Yang extended Xpath functions: re-match, deref, derived-from, derived-from-or-self, enum-value, bit-is-set
 
 Restrictions on Yang types are as follows:
 - Submodules cannot re-use a prefix in an import statement that is already used for another imported module in the module that the submodule belongs to. (see https://github.com/clicon/clixon/issues/60)

apps/backend/backend_commit.c

@@ -273,7 +273,7 @@ startup_validate(clicon_handle  h,
     int                 retval = -1;
     yang_spec          *yspec;
     int                 ret;
-    cxobj              *xms = NULL;
+    modstate_diff_t    *msd = NULL;
     transaction_data_t *td = NULL;
 
     /* Handcraft a transition with only target and add trees */
@@ -282,11 +282,17 @@ startup_validate(clicon_handle  h,
     /* 2. Parse xml trees 
      * This is the state we are going to 
      * Note: xmsdiff contains non-matching modules
+     * Only if CLICON_XMLDB_MODSTATE is enabled 
      */
-    if (xmldb_get(h, db, "/", 1, &td->td_target, &xms) < 0)
+    if (clicon_option_bool(h, "CLICON_XMLDB_MODSTATE"))
+	if ((msd = modstate_diff_new()) == NULL)
+	    goto done;
+    if (xmldb_get(h, db, "/", 1, &td->td_target, msd) < 0)
 	goto done;
-    if (xms && clixon_plugin_upgrade(h, xms) < 0)
+    if ((ret = clixon_module_upgrade(h, td->td_target, msd, cbret)) < 0)
 	goto done;
+    if (ret == 0)
+	goto fail;
 
     /* Handcraft transition with with only add tree */
     if (cxvec_append(td->td_target, &td->td_avec, &td->td_alen) < 0) 
@@ -318,8 +324,8 @@ startup_validate(clicon_handle  h,
  done:
     if (td)
 	transaction_free(td);
-    if (xms)
-	xml_free(xms);
+    if (msd)
+	modstate_diff_free(msd);
     return retval;
  fail: /* cbret should be set */
     if (cbuf_len(cbret)==0){	

apps/backend/backend_main.c

@@ -96,7 +96,10 @@ backend_terminate(clicon_handle h)
     clicon_debug(1, "%s", __FUNCTION__);
     if ((ss = clicon_socket_get(h)) != -1)
 	close(ss);
-    modules_state_cache_set(h, NULL);
+    if ((x = clicon_module_state_get(h)) != NULL)
+	xml_free(x);
+    if ((x = clicon_yang_changelog_get(h)) != NULL)
+	xml_free(x);
     if ((yspec = clicon_dbspec_yang(h)) != NULL)
 	yspec_free(yspec);
     if ((yspec = clicon_config_yang(h)) != NULL)
@@ -639,6 +642,11 @@ main(int    argc,
     if (backend_rpc_init(h) < 0)
 	goto done;
 
+    /* Must be after netconf_module_load, but before startup code */
+    if (clicon_option_bool(h, "CLICON_YANG_CHANGELOG"))
+	if (clixon_yang_changelog_init(h) < 0)
+	    goto done;
+
     /* Save modules state of the backend (server). Compare with startup XML */
     if (startup_module_state(h, yspec) < 0)
 	goto done;

apps/backend/backend_plugin.c

@@ -156,31 +156,6 @@ clixon_plugin_statedata(clicon_handle    h,
     return retval;
 }
 
-/*! Call configuration upgrade routines in backend plugins
- * @param[in]  h    Clicon handle
- * @param[in]  xms  XML tree of module state differences
- * @retval     0    OK
- * @retval    -1    Error in one (first) of user callbacks
- */
-int
-clixon_plugin_upgrade(clicon_handle       h,
-		      cxobj              *xmodst)
-{
-    int             retval = -1;
-    clixon_plugin  *cp = NULL;
-    upgrade_cb_t *fn;          /* Plugin configuration upgrade fn */
-
-    while ((cp = clixon_plugin_each(h, cp)) != NULL) {
-	if ((fn = cp->cp_api.ca_upgrade) == NULL)
-	    continue;
-	if (fn(h, xmodst) < 0)
-	    goto done; 
-    }
-    retval = 0;
- done:
-    return retval;
-}
-
 /*! Create and initialize transaction */
 transaction_data_t *
 transaction_new(void)

apps/backend/backend_plugin.h

@@ -72,7 +72,6 @@ int backend_plugin_initiate(clicon_handle h);
 int clixon_plugin_reset(clicon_handle h, char *db);
 
 int clixon_plugin_statedata(clicon_handle h, yang_spec *yspec, char *xpath, cxobj **xtop);
-int clixon_plugin_upgrade(clicon_handle h, cxobj *xmodst);
 transaction_data_t * transaction_new(void);
 int transaction_free(transaction_data_t *);
 

datastore/clixon_xmldb_text.c

@@ -364,12 +364,11 @@ singleconfigroot(cxobj  *xt,
 
 static int
 text_get_nocache(struct text_handle *th,
-		 const char   *db, 
-		 char         *xpath,
-		 int           config,
-		 cxobj       **xtop,
-		 cxobj       **xms)
-
+		 const char         *db, 
+		 char               *xpath,
+		 int                 config,
+		 cxobj             **xtop,
+		 modstate_diff_t    *msd)
 {
     int             retval = -1;
     char           *dbfile = NULL;
@@ -385,7 +384,6 @@ text_get_nocache(struct text_handle *th,
 	clicon_err(OE_YANG, ENOENT, "No yang spec");
 	goto done;
     }
-
     if (text_db2file(th, db, &dbfile) < 0)
 	goto done;
     if (dbfile==NULL){
@@ -559,7 +557,7 @@ xml_copy_marked(cxobj *x0,
  * @param[in]  th    Datastore text handle
  * @param[in]  yspec Top-level yang spec 
  * @param[in]  xt    XML tree
- * @param[out] xmsp  If set, return modules-state differences
+ * @param[out] msd   If set, return modules-state differences
  *
  * Read mst (module-state-tree) from xml tree (if any) and compare it with 
  * the system state mst.
@@ -575,7 +573,7 @@ static int
 text_read_modstate(struct text_handle *th,
 		   yang_spec          *yspec,
 		   cxobj              *xt,
-		   cxobj             **xmsp)
+		   modstate_diff_t    *msd)
 {
     int retval = -1;
     cxobj *xmodst;
@@ -585,16 +583,19 @@ text_read_modstate(struct text_handle *th,
     char  *name; /* module name */
     char  *mrev; /* file revision */
     char  *srev; /* system revision */
-    cxobj *xmsd = NULL; /* Local modules-state diff tree */
 
     if ((xmodst = xml_find_type(xt, NULL, "modules-state", CX_ELMNT)) == NULL){
 	/* 1) There is no modules-state info in the file */
     }
-    else if (th->th_modst){
-	/* Create a diff tree */
-	if (xml_parse_string("<modules-state xmlns=\"urn:ietf:params:xml:ns:yang:ietf-yang-library\"/>", yspec, &xmsd) < 0)
+    else if (th->th_modst && msd){
+	/* Create diff trees */
+	if (xml_parse_string("<modules-state xmlns=\"urn:ietf:params:xml:ns:yang:ietf-yang-library\"/>", yspec, &msd->md_del) < 0)
+	    goto done;
+	if (xml_rootchild(msd->md_del, 0, &msd->md_del) < 0) 
 	    goto done;
-	if (xml_rootchild(xmsd, 0, &xmsd) < 0) 
+	if (xml_parse_string("<modules-state xmlns=\"urn:ietf:params:xml:ns:yang:ietf-yang-library\"/>", yspec, &msd->md_mod) < 0)
+	    goto done;
+	if (xml_rootchild(msd->md_mod, 0, &msd->md_mod) < 0) 
 	    goto done;
 
 	/* 3) For each module state m in the file */
@@ -608,7 +609,7 @@ text_read_modstate(struct text_handle *th,
 		//		fprintf(stderr, "%s: Module %s: not in system\n", __FUNCTION__, name);
 		if ((xm2 = xml_dup(xm)) == NULL)
 		    goto done;
-		if (xml_addsub(xmsd, xm2) < 0)
+		if (xml_addsub(msd->md_del, xm2) < 0)
 		    goto done;
 		continue;
 	    }
@@ -626,7 +627,7 @@ text_read_modstate(struct text_handle *th,
 		//		fprintf(stderr, "%s: Module %s: file \"%s\" and system \"%s\" revisions do not match\n", __FUNCTION__, name, mrev, srev);
 		if ((xm2 = xml_dup(xm)) == NULL)
 		    goto done;
-		if (xml_addsub(xmsd, xm2) < 0)
+		if (xml_addsub(msd->md_mod, xm2) < 0)
 		    goto done;
 	    }
 	}
@@ -639,14 +640,8 @@ text_read_modstate(struct text_handle *th,
 	if (xml_purge(xmodst) < 0)
 	    goto done;
     }
-    if (xmsp){
-	*xmsp = xmsd;
-	xmsd = NULL;
-    }
     retval = 0;
  done:
-    if (xmsd)
-	xml_free(xmsd);
     return retval;
 }
 
@@ -655,14 +650,14 @@ text_read_modstate(struct text_handle *th,
  * @param[in]  db    Symbolic database name, eg "candidate", "running"
  * @param[in]  yspec Top-level yang spec
  * @param[out] xp    XML tree read from file
- * @param[out] xmsp  If set, return modules-state differences
+ * @param[out] msd    If set, return modules-state differences
  */
 static int
 text_readfile(struct text_handle *th,
 	      const char         *db,
 	      yang_spec          *yspec,
 	      cxobj             **xp,
-	      cxobj             **xmsp)
+	      modstate_diff_t    *msd)
 {
     int    retval = -1;
     cxobj *x0 = NULL;
@@ -702,7 +697,7 @@ text_readfile(struct text_handle *th,
     /* From Clixon 3.10,datastore files may contain module-state defining
      * which modules are used in the file. 
      */
-    if (text_read_modstate(th, yspec, x0, xmsp) < 0)
+    if (text_read_modstate(th, yspec, x0, msd) < 0)
 	goto done;
     if (xp){
 	*xp = x0;
@@ -728,18 +723,18 @@ text_readfile(struct text_handle *th,
  * @param[in]  xpath  String with XPATH syntax. or NULL for all
  * @param[in]  config If set only configuration data, else also state
  * @param[out] xret   Single return XML tree. Free with xml_free()
- * @param[out] xms    If set, return modules-state differences
+ * @param[out] msd    If set, return modules-state differences
  * @retval     0      OK
  * @retval     -1     Error
  * @see xmldb_get  the generic API function
  */
 static int
 text_get_cache(struct text_handle *th,
-	       const char   *db, 
-	       char         *xpath,
-	       int           config,
-	       cxobj       **xtop,
-	       cxobj       **xms)
+	       const char         *db, 
+	       char               *xpath,
+	       int                 config,
+	       cxobj             **xtop,
+	       modstate_diff_t    *msd)
 {
     int             retval = -1;
     yang_spec      *yspec;
@@ -759,7 +754,7 @@ text_get_cache(struct text_handle *th,
     de = hash_value(th->th_dbs, db, NULL);
     if (de == NULL || de->de_xml == NULL){ /* Cache miss, read XML from file */
 	/* If there is no xml x0 tree (in cache), then read it from file */
-	if (text_readfile(th, db, yspec, &x0t, xms) < 0)
+	if (text_readfile(th, db, yspec, &x0t, msd) < 0)
 	    goto done;
 	/* XXX: should we validate file if read from disk? 
 	 * Argument against: we may want to have a semantically wrong file and wish
@@ -828,24 +823,25 @@ text_get_cache(struct text_handle *th,
  * @param[in]  xpath  String with XPATH syntax. or NULL for all
  * @param[in]  config If set only configuration data, else also state
  * @param[out] xret   Single return XML tree. Free with xml_free()
+ * @param[out] msd    If set, return modules-state differences
  * @retval     0      OK
  * @retval     -1     Error
  * @see xmldb_get  the generic API function
  */
 int
-text_get(xmldb_handle  xh,
-	 const char   *db, 
-	 char         *xpath,
-	 int           config,
-	 cxobj       **xtop,
-	 cxobj       **xms)
+text_get(xmldb_handle     xh,
+	 const char      *db, 
+	 char            *xpath,
+	 int              config,
+	 cxobj          **xtop,
+	 modstate_diff_t *msd)
 {
     struct text_handle *th = handle(xh);
 
     if (th->th_cache)
-	return text_get_cache(th, db, xpath, config, xtop, xms);
+	return text_get_cache(th, db, xpath, config, xtop, msd);
     else
-	return text_get_nocache(th, db, xpath, config, xtop, xms);
+	return text_get_nocache(th, db, xpath, config, xtop, msd);
 }
 
 /*! Modify a base tree x0 with x1 with yang spec y according to operation op
@@ -1019,7 +1015,7 @@ text_modify(struct text_handle *th,
 	       can be modified in its entirety only.
 	       Any "operation" attributes present on subelements of an anyxml 
 	       node are ignored by the NETCONF server.*/
-	    if (y0->yn_keyword == Y_ANYXML){
+	    if (y0->yn_keyword == Y_ANYXML || y0->yn_keyword == Y_ANYDATA){
 		if (op == OP_NONE)
 		    break;
 		if (op==OP_MERGE && !permit && xnacm){

datastore/clixon_xmldb_text.h

@@ -39,7 +39,7 @@
 /*
  * Prototypes
  */
-int text_get(xmldb_handle h, const char *db, char *xpath, int config, cxobj **xtop, cxobj **xms);
+int text_get(xmldb_handle h, const char *db, char *xpath, int config, cxobj **xtop, modstate_diff_t *xms);
 int text_put(xmldb_handle h, const char *db, enum operation_type op, cxobj *xt, char *username, cbuf *cbret);
 int text_dump(FILE *f, char *dbfilename, char *rxkey);
 int text_copy(xmldb_handle h, const char *from, const char *to);