Remove unreleased dependency tags when library is shipped (#45427)

Remove unreleased dependency tags when library is shipped

Author: alzimmermsft

eng/versioning/set_versions.py

@@ -90,7 +90,7 @@ def set_dev_zero_version(build_type, build_qualifier):
                 newlines.append(module.string_for_version_file())
                 continue
 
-            if hasattr(module, 'current'):
+            if module.current != None:
 
                 if 'alpha' in module.current:
                     newlines.append(module.string_for_version_file())
@@ -143,7 +143,7 @@ def update_versions_file_for_nightly_devops(build_type, build_qualifier, artifac
                     continue
                 if library_to_update == module.name:
                     artifact_found = True
-                    if hasattr(module, 'current'):
+                    if module.current != None:
                         set_both = False
                         # In the case where the current and dependency are both equal then both
                         # need to be updated. In theory, this should only really happen when a
@@ -223,7 +223,7 @@ def prep_version_file_for_source_testing(build_type, project_list):
                 newlines.append(raw_line)
             else:
                 module = CodeModule(stripped_line)
-                if hasattr(module, 'current') and not module.current == module.dependency:
+                if module.current != None and not module.current == module.dependency:
                     # If the project list is passed in, only prep the versions for from source
                     # build for those modules. This is the case specifically for patch release.
                     if project_list_identifiers is not None:
@@ -279,7 +279,7 @@ def increment_or_set_library_version(build_type, artifact_id, group_id, new_vers
                 # to use based on what has been released to Maven central. If "beta.1"
                 # exists in Maven central then use "beta.2" and so on.
                 # https://github.com/Azure/azure-sdk/blob/main/docs/policies/releases.md#java
-                if module.name == library_to_update and hasattr(module, 'current'):
+                if module.name == library_to_update and module.current != None:
                     artifact_found = True
                     if new_version is None:
                         vmatch = version_regex_named.match(module.current)
@@ -398,7 +398,7 @@ def verify_current_version_of_artifact(build_type, artifact_id, group_id):
                 # of the following:
                 # <major>.<minor>.<patch/hotfix>
                 # <major>.<minor>.<patch/hotfix>-beta.<prerelease>
-                if module.name == library_to_update and hasattr(module, 'current'):
+                if module.name == library_to_update and module.current != None:
                     artifact_found = True
                     vmatch = version_regex_named.match(module.current)
                     temp_ver = '{}.{}.{}'.format(vmatch.group('major'), vmatch.group('minor'), vmatch.group('patch'))

eng/versioning/update_versions.py

@@ -36,7 +36,7 @@
 import re
 import sys
 import time
-import traceback
+from typing import Dict
 from utils import BuildType
 from utils import CodeModule
 from utils import load_version_map_from_file
@@ -53,11 +53,11 @@
 
 exception_list = []
 
-def update_versions(update_type, version_map, ext_dep_map, target_file, skip_readme, auto_version_increment, library_array):
+def update_versions(update_type, version_map: Dict[str, CodeModule], ext_dep_map, target_file, skip_readme, auto_version_increment, library_array):
 
     newlines = []
     repl_open, repl_thisline, file_changed, is_include = False, False, False, False
-    print('processing: ' + target_file)
+    print('processing: ' + os.path.normpath(target_file))
     try:
         with open(target_file, encoding='utf-8') as f:
             for line in f:
@@ -105,8 +105,12 @@ def update_versions(update_type, version_map, ext_dep_map, target_file, skip_rea
                     elif version_type == 'dependency':
                         try:
                             module = version_map[module_name]
-                            new_version = module.dependency
-                            newline = re.sub(version_regex_str_no_anchor, new_version, line)
+                            if module.is_unreleased and module.replace_unreleased_dependency:
+                                to_replace_module = version_map[module.name[len('unreleased_'):]]
+                                newline = re.sub(version_regex_str_no_anchor, to_replace_module.dependency, line)
+                                newline = newline.replace(module.name, to_replace_module.name)
+                            else:
+                                newline = re.sub(version_regex_str_no_anchor, module.dependency, line)
                         except (KeyError, AttributeError):
                             # This should never happen unless the version file is malformed
                             raise ValueError('Module: {0} does not have a dependency version.\nFile={1}\nLine={2}'.format(module_name, target_file, line))
@@ -170,7 +174,7 @@ def update_changelog(pom_file, is_increment, library_array):
         xml_groupId = xml_root.find('{http://maven.apache.org/POM/4.0.0}groupId')
         library = xml_groupId.text + ":" + xml_artifactId.text
         if len(library_array) == 0 or library in library_array:
-            script = os.path.join(".", "eng", "common", "scripts", "Update-ChangeLog.ps1")
+            script = os.path.join(os.path.dirname(__file__), '..', "common", "scripts", "Update-ChangeLog.ps1")
             commands = [
                 "pwsh",
                 script,
@@ -207,23 +211,23 @@ def display_version_info(version_map):
     for value in version_map.values():
         print(value)
 
-def update_versions_all(update_type, build_type, target_file, skip_readme, auto_version_increment, library_array, version_overrides, include_perf_tests):
-    version_map = {}
-    ext_dep_map = {}
+def update_versions_all(update_type, build_type, target_file, skip_readme, auto_version_increment: bool, library_array, version_overrides, include_perf_tests):
+    version_map: Dict[str, CodeModule] = {}
+    ext_dep_map: Dict[str, CodeModule] = {}
     # Load the version and/or external dependency file for the given UpdateType
     # into the verion_map. If UpdateType.all is selected then versions for both
     # the libraries and external dependencies are being updated.
     if update_type == UpdateType.library or update_type == UpdateType.all:
-        version_file = os.path.normpath('eng/versioning/version_' + build_type.name + '.txt')
-        load_version_map_from_file(version_file, version_map)
+        version_file = os.path.join(os.path.dirname(__file__), 'version_' + build_type.name + '.txt')
+        load_version_map_from_file(version_file, version_map, auto_version_increment)
 
     if update_type == UpdateType.external_dependency or update_type == UpdateType.all:
-        dependency_file = os.path.normpath('eng/versioning/external_dependencies.txt')
+        dependency_file = os.path.join(os.path.dirname(__file__), 'external_dependencies.txt')
         load_version_map_from_file(dependency_file, ext_dep_map)
 
     if version_overrides and not version_overrides.startswith('$'):
         # Azure DevOps passes '$(VersionOverrides)' when the variable value is not set
-        load_version_overrides("eng/versioning/supported_external_dependency_versions.json", ext_dep_map, version_overrides)
+        load_version_overrides(os.path.join(os.path.dirname(__file__), 'supported_external_dependency_versions.json'), ext_dep_map, version_overrides)
 
     # The dependency files are always loaded but reporting their information is based on the update type.
     if update_type == UpdateType.library or update_type == UpdateType.all:
@@ -237,7 +241,7 @@ def update_versions_all(update_type, build_type, target_file, skip_readme, auto_
     if target_file:
         update_versions(update_type, version_map, ext_dep_map, target_file, skip_readme, auto_version_increment, library_array)
     else:
-        for root, _, files in os.walk("."):
+        for root, _, files in os.walk(os.path.join(os.path.dirname(__file__), '..', '..')):
             for file_name in files:
                 file_path = root + os.sep + file_name
                 if (file_name.endswith('.md') and not skip_readme) or (file_name.startswith('pom') and file_name.endswith('.xml')):
@@ -254,7 +258,7 @@ def update_versions_all(update_type, build_type, target_file, skip_readme, auto_
     if not target_file and BuildType.none != build_type:
         # the good thing here is that the java files only contain library versions, not
         # external versions
-        version_java_file = os.path.normpath('eng/versioning/version_' + build_type.name + '_java_files.txt')
+        version_java_file = os.path.join(os.path.dirname(__file__), 'version_' + build_type.name + '_java_files.txt')
 
         if os.path.exists(version_java_file):
             with open(version_java_file) as f:

eng/versioning/utils.py

@@ -6,6 +6,7 @@
 
 from enum import Enum
 import re
+from typing import Dict
 from subprocess import check_call, CalledProcessError
 
 include_update_marker = re.compile(r'\{x-include-update;([^;]+);([^}]+)\}')
@@ -65,6 +66,17 @@ def __str__(self):
         return self.value
 
 class CodeModule:
+    name: str
+    group_id: str
+    artifact_id: str
+    dependency: str
+    current: str = None # current version is optional, only for library versions
+    external_dependency: str = None # external dependency version is optional, only for external dependencies
+    update_type: UpdateType
+    is_beta: bool = False # beta_ is a prefix for unreleased dependencies, may not exist
+    is_unreleased: bool = False # unreleased_ is a prefix for unreleased dependencies, may not exist
+    replace_unreleased_dependency: bool = False # used to indicate that the unreleased dependency should be replaced with the dependency version
+
     def __init__(self, module_str):
         # For library versions there will be up to 3 items resulting from the split
         # which will be module name, dependency version and current version. For
@@ -77,7 +89,9 @@ def __init__(self, module_str):
         self.artifact_id = items[0].split(':')[1]
 
         if len(items) == 2:
-            if self.group_id.startswith('unreleased_') or self.group_id.startswith('beta_'):
+            self.is_beta = self.group_id.startswith('beta_')
+            self.is_unreleased = self.group_id.startswith('unreleased_')
+            if self.is_beta or self.is_unreleased:
                 self.dependency = items[1].strip()
                 self.update_type = UpdateType.library
             else:
@@ -95,18 +109,18 @@ def __init__(self, module_str):
     # overridden string primarily used for error reporting
     def __str__(self):
         # current may or may not exist
-        if hasattr(self, 'external_dependency'):
+        if self.external_dependency != None:
             return self.name + ': External Dependency version=' + self.external_dependency
-        try:
+        if self.current != None:
             return self.name + ': Dependency version=' + self.dependency + ': Current version=' + self.current
-        except AttributeError:
+        else:
             return self.name + ': Dependency version=' + self.dependency
 
     # return the CodeModule string formatted for a version file
     def string_for_version_file(self):
-        try:
+        if self.current != None:
             return self.name + ';' + self.dependency + ';' + self.current + '\n'
-        except AttributeError:
+        else:
             return self.name + ';' + self.dependency + '\n'
 
     # return the CodeModule string formatted for a allowlist include entry
@@ -115,7 +129,7 @@ def string_for_version_file(self):
     # would be treated as that version and above. For example:
     # <groupId>:<artifactId>:1.2 would be treated as 1.2 and above or equivalent to [1.2,)
     def string_for_allowlist_include(self):
-        if hasattr(self, 'external_dependency'):
+        if self.external_dependency != None:
             temp = self.name
             # This is necessary to deal with the fact that external_dependencies can have
             # '_' in them if they're an external dependency exception. Since the allowlist
@@ -159,25 +173,53 @@ def run_check_call(
                 return err
 
 
-def load_version_map_from_file(the_file, version_map):
+def load_version_map_from_file(the_file, version_map: Dict[str, CodeModule], auto_increment_version=False):
+    newlines = []
+    file_changed = False
     with open(the_file) as f:
         for raw_line in f:
             stripped_line = raw_line.strip()
             if not stripped_line or stripped_line.startswith('#'):
+                newlines.append(raw_line)
                 continue
             module = CodeModule(stripped_line)
             # verify no duplicate entries
             if (module.name in version_map):
                 raise ValueError('Version file: {0} contains a duplicate entry: {1}'.format(the_file, module.name))
             # verify that if the module is beta_ or unreleased_ that there's a matching non-beta_ or non-unreleased_ entry
-            if (module.name.startswith('beta_') or module.name.startswith('unreleased_')):
+            is_beta = module.name.startswith('beta_')
+            is_unreleased = module.name.startswith('unreleased_')
+            if (is_beta or is_unreleased):
                 tempName = module.name
-                if tempName.startswith('beta_'):
+                if is_beta:
                     tempName = module.name[len('beta_'):]
                 else:
                     tempName = module.name[len('unreleased_'):]
                 # if there isn't a non beta or unreleased entry then raise an issue
                 if tempName not in version_map:
                     raise ValueError('Version file: {0} does not contain a non-beta or non-unreleased entry for beta_/unreleased_ library: {1}'.format(the_file, module.name))
+                
+                # Unreleased dependencies have a few additional checks.
+                # 1. If 'auto_increment_version' is true, check if the version matches the dependency version of the library.
+                #    If the unreleased dependency version is the same as the dependency version, flag the unreleased dependency for replacement.
+                #    This flag will indicate to the update script that the 'unreleased_*' tag should be replaced with just '*', ex 'unreleased_core' -> 'core'.
+                # 2. Check to see that the version matches the current version of the library.
+                #    If it isn't raise an error as unreleased dependencies should match the current version of the library.
+                if is_unreleased:
+                    non_unreleased_module = version_map[tempName]
+                    if auto_increment_version and module.dependency == non_unreleased_module.dependency:
+                        module.replace_unreleased_dependency = True
+                        file_changed = True
+                    elif module.dependency != non_unreleased_module.current:
+                        raise ValueError('Version file: {0} contains an unreleased dependency: {1} with a dependency version of {2} which does not match the current version of the library it represents: {3}'.format(the_file, module.name, module.dependency, version_map[tempName].current))
+            
+            if not module.replace_unreleased_dependency:
+                newlines.append(raw_line)
+
             version_map[module.name] = module
+
+    if file_changed:
+        with open(the_file, 'w', encoding='utf-8') as f:
+            for line in newlines:
+                f.write(line)
 

sdk/appconfiguration/azure-data-appconfiguration-v2/pom.xml

@@ -56,14 +56,14 @@ Code generated by Microsoft (R) TypeSpec Code Generator.
     <dependency>
       <groupId>com.azure.v2</groupId>
       <artifactId>azure-core</artifactId>
-      <version>2.0.0-beta.1</version> <!-- {x-version-update;com.azure.v2:azure-core;dependency} -->
+      <version>2.0.0-beta.1</version> <!-- {x-version-update;unreleased_com.azure.v2:azure-core;dependency} -->
     </dependency>
 
     <!-- Test scope -->
     <dependency>
       <groupId>com.azure.v2</groupId>
       <artifactId>azure-identity</artifactId>
-      <version>2.0.0-beta.1</version> <!-- {x-version-update;com.azure.v2:azure-identity;dependency} -->
+      <version>2.0.0-beta.1</version> <!-- {x-version-update;unreleased_com.azure.v2:azure-identity;dependency} -->
       <scope>test</scope>
     </dependency>
 

sdk/identity-v2/pom.xml

@@ -1,21 +1,15 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <!-- Copyright (c) Microsoft Corporation. All rights reserved.
      Licensed under the MIT License. -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
-
   <groupId>com.azure.v2</groupId>
-
   <artifactId>azure-identity-v2-service</artifactId>
-
   <packaging>pom</packaging>
-
-  <version>1.0.0</version>
-  <!-- Need not change for every release-->
+  <version>1.0.0</version> <!-- Need not change for every release-->
 
   <modules>
     <module>azure-identity</module>
   </modules>
-
 </project>

sdk/identity/pom.xml

@@ -1,18 +1,13 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <!-- Copyright (c) Microsoft Corporation. All rights reserved.
      Licensed under the MIT License. -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
-
   <groupId>com.azure</groupId>
-
   <artifactId>azure-identity-service</artifactId>
-
   <packaging>pom</packaging>
-
-  <version>1.0.0</version>
-  <!-- Need not change for every release-->
+  <version>1.0.0</version> <!-- Need not change for every release-->
 
   <modules>
     <module>azure-identity</module>
@@ -21,5 +16,4 @@
     <module>azure-identity-broker</module>
     <module>azure-identity-broker-samples</module>
   </modules>
-
 </project>