Merge pull request #178 from xtreme1-io/dev

Release v0.8.0

Author: JzDmUrMWzezh

.ops/.gitlab-ci.yml

@@ -4,4 +4,4 @@ include:
 
 variables:
   APP_NAME: "xtreme1"
-  APP_VERSION: "0.7.3"
+  APP_VERSION: "0.8"

README.md

@@ -1,7 +1,7 @@
 <div align="center">
 <img width="386" alt="Xtreme1 logo" src="https://user-images.githubusercontent.com/84139543/190300943-98da7d5c-bd67-4074-a94f-b7405d29fb90.png">
 
-![](https://img.shields.io/badge/Release-v0.7.3-green) 
+![](https://img.shields.io/badge/Release-v0.8-green) 
 ![](https://img.shields.io/badge/License-Apache%202.0-blueviolet)
 [![Slack](https://img.shields.io/badge/Join-Slack-orange.svg?logo=slack)](https://join.slack.com/t/xtreme1group/shared_invite/zt-1jhk36uzr-NpdpYXeQAEHN6rYJy5_6pg)
 [![Twitter](https://img.shields.io/badge/Follow-Twitter-blue)](https://twitter.com/Xtreme1io)
@@ -66,8 +66,8 @@ Image Data Curation (Visualizing & Debug)  - [MobileNetV3](https://github.com/xi
 Download the latest release package and unzip it.
 
 ```bash
-wget https://github.com/xtreme1-io/xtreme1/releases/download/v0.7.3/xtreme1-v0.7.3.zip
-unzip -d xtreme1-v0.7.3 xtreme1-v0.7.3.zip
+wget https://github.com/xtreme1-io/xtreme1/releases/download/v0.8/xtreme1-v0.8.zip
+unzip -d xtreme1-v0.8 xtreme1-v0.8.zip
 ```
 
 ## Start all services

backend/Dockerfile

@@ -9,9 +9,9 @@ RUN apt update && \
     apt install -y iputils-ping curl wget netcat python3 python3-pip git
 RUN pip3 install --upgrade --force-reinstall git+https://github.com/xtreme1-io/xtreme1-sdk.git@d0cf4cc
 WORKDIR /app
-COPY --from=build /build/target/xtreme1-backend-0.7.3-SNAPSHOT.jar ./app.jar
+COPY --from=build /build/target/xtreme1-backend-0.8-SNAPSHOT.jar ./app.jar
 RUN mkdir -p config
-RUN wget 'https://basicai-asset.s3.us-west-2.amazonaws.com/xtreme1/xtreme1-lidar-fusion-trial.zip' -O xtreme1-lidar-fusion-trial.zip
-RUN wget 'https://basicai-asset.s3.us-west-2.amazonaws.com/xtreme1/xtreme1-image-trial.zip' -O xtreme1-image-trial.zip
+RUN wget 'https://github.com/xtreme1-io/asset/raw/main/datasets/xtreme1-lidar-fusion-trial.zip' -O xtreme1-lidar-fusion-trial.zip
+RUN wget 'https://github.com/xtreme1-io/asset/raw/main/datasets/xtreme1-image-trial.zip' -O xtreme1-image-trial.zip
 EXPOSE 8080
 CMD java -jar app.jar

backend/README.md

@@ -92,7 +92,7 @@ cd backend
 mvn package
 
 # Using local configuration to start application.
-java -Dspring.profiles.active=local -jar target/xtreme1-backend-0.7.3-SNAPSHOT.jar
+java -Dspring.profiles.active=local -jar target/xtreme1-backend-0.8-SNAPSHOT.jar
 ```
 
 Now you can access the backend service at `http://localhost:8080/`.

backend/pom.xml

@@ -13,7 +13,7 @@
 
     <groupId>ai.basic</groupId>
     <artifactId>xtreme1-backend</artifactId>
-    <version>0.7.3-SNAPSHOT</version>
+    <version>0.8-SNAPSHOT</version>
     <name>Xtreme1 Backend</name>
     <description></description>
 
@@ -29,6 +29,7 @@
         <okhttp.version>4.8.1</okhttp.version>
         <fegin.version>11.7</fegin.version>
         <thumbnail.version>0.4.17</thumbnail.version>
+        <webp-imageio-version>0.1.6</webp-imageio-version>
     </properties>
 
     <dependencies>
@@ -164,6 +165,12 @@
             <version>${thumbnail.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>org.sejda.imageio</groupId>
+            <artifactId>webp-imageio</artifactId>
+            <version>${webp-imageio-version}</version>
+        </dependency>
+
     </dependencies>
 
     <build>

backend/src/main/java/ai/basic/x1/adapter/api/config/CommonConfig.java

@@ -192,5 +192,20 @@ public ModelRunRecordUseCase modelRunRecordUseCase() {
         return new ModelRunRecordUseCase();
     }
 
+    @Bean
+    public UploadDataUseCase uploadDataUseCase() {
+        return new UploadDataUseCase();
+    }
+
+    @Bean
+    public PointCloudUploadUseCase pointCloudUploadUseCase() {
+        return new PointCloudUploadUseCase();
+    }
+
+    @Bean
+    public ImageUploadUseCase imageUploadUseCase() {
+        return new ImageUploadUseCase();
+    }
+
 
 }

backend/src/main/java/ai/basic/x1/adapter/api/controller/DataFlowController.java

@@ -29,8 +29,8 @@ public void markAsValid(@PathVariable Long dataId){
         dataFlowUseCase.changeDataStatus(dataId, DataStatusEnum.VALID);
     }
 
-    @PostMapping("/submit/{dataId}")
-    public void submit(@PathVariable Long dataId){
-        dataFlowUseCase.submit(dataId);
+    @PostMapping("/submit/{itemId}")
+    public void submit(@PathVariable Long itemId){
+        dataFlowUseCase.submit(itemId);
     }
 }

backend/src/main/java/ai/basic/x1/adapter/api/controller/DataInfoController.java

@@ -33,6 +33,7 @@
 import java.io.InputStreamReader;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 
 /**
@@ -69,12 +70,15 @@ public class DataInfoController extends BaseDatasetController {
     @Autowired
     private DataClassificationOptionUseCase dataClassificationOptionUseCase;
 
+    @Autowired
+    protected UploadDataUseCase uploadDataUseCase;
+
     @PostMapping("upload")
     public String upload(@RequestBody @Validated DataInfoUploadDTO dto, @LoggedUser LoggedUserDTO userDTO) throws IOException {
         var dataInfoUploadBO = DefaultConverter.convert(dto, DataInfoUploadBO.class);
         assert dataInfoUploadBO != null;
         dataInfoUploadBO.setUserId(userDTO.getId());
-        return String.valueOf(dataInfoUsecase.upload(dataInfoUploadBO));
+        return String.valueOf(uploadDataUseCase.upload(dataInfoUploadBO));
     }
 
     @GetMapping("findUploadRecordBySerialNumbers")
@@ -289,4 +293,10 @@ public List<DatasetModelResultDTO> getDataModelRunResult(@PathVariable Long data
         return DefaultConverter.convert(dataAnnotationObjectUseCase.getDataModelRunResult(dataId), DatasetModelResultDTO.class);
     }
 
+    @GetMapping("/getDataIdBySceneIds")
+    public Map<Long, List<Long>> getDataIdBySceneIds(@NotNull(message = "datasetId cannot be null") @RequestParam(required = false) Long datasetId,
+                                                     @NotEmpty(message = "sceneIds cannot be null") @RequestParam(required = false) List<Long> sceneIds) {
+        return dataInfoUsecase.getDataIdBySceneIds(datasetId, sceneIds);
+    }
+
 }

backend/src/main/java/ai/basic/x1/adapter/api/controller/DatasetController.java

@@ -93,9 +93,7 @@ public Page<DatasetDTO> findByPage(@RequestParam(defaultValue = "1") Integer pag
             }
             if (CollectionUtil.isNotEmpty(datasetBO.getDatas())) {
                 var dataInfoDTOS = new ArrayList<DataInfoDTO>();
-                datasetBO.getDatas().forEach(dataInfoBO -> {
-                    dataInfoDTOS.add(convertDataInfoDTO(dataInfoBO));
-                });
+                datasetBO.getDatas().forEach(dataInfoBO -> dataInfoDTOS.add(convertDataInfoDTO(dataInfoBO)));
                 datasetDTO.setDatas(dataInfoDTOS);
             }
             return datasetDTO;

backend/src/main/java/ai/basic/x1/adapter/api/job/converter/ModelCocoRequestConverter.java

@@ -7,6 +7,8 @@
 
 import java.util.List;
 
+import static ai.basic.x1.util.Constants.FILE;
+
 /**
  * @author zhujh
  */
@@ -22,7 +24,13 @@ public static ImageDetectionReqDTO convert(ModelMessageBO message) {
         if (CollUtil.isEmpty(fileNodes)) {
             throw new IllegalArgumentException("file is not found");
         }
-        String url = fileNodes.get(0).getFile().getInternalUrl();
+        var fileNodeBO = fileNodes.get(0);
+        String url;
+        if (fileNodeBO.getType().equals(FILE)) {
+            url = fileNodeBO.getFile().getInternalUrl();
+        } else {
+            url = fileNodeBO.getFiles().get(0).getFile().getInternalUrl();
+        }
         if (StrUtil.isEmpty(url)) {
             throw new IllegalArgumentException("file url is empty");
         }

backend/src/main/java/ai/basic/x1/adapter/api/job/converter/PointCloudDetectionModelReqConverter.java

@@ -49,7 +49,7 @@ private static void traverFile(DataInfoBO.FileNodeBO fileNodeBO, DataInfo dataIn
             }
             String prePath = subPaths[subPaths.length - 2];
             //images
-            if (prePath.startsWith(POINT_CLOUD_IMG)) {
+            if (prePath.startsWith(CAMERA_IMAGE)) {
                 if (CollUtil.isEmpty(dataInfo.getImageUrls())) {
                     List<String> imageUrlList = new ArrayList<>();
                     imageUrlList.add(fileNodeBO.getFile().getInternalUrl());
@@ -59,7 +59,7 @@ private static void traverFile(DataInfoBO.FileNodeBO fileNodeBO, DataInfo dataIn
                 }
             }
             //pcd
-            if (prePath.startsWith(POINT_CLOUD)) {
+            if (prePath.startsWith(LIDAR_POINT_CLOUD)) {
                 dataInfo.setPointCloudUrl(getFileBO(fileNodeBO.getFile()).getInternalUrl());
             }
             //cameraConfig

backend/src/main/java/ai/basic/x1/adapter/dto/DataAnnotateDTO.java

@@ -1,5 +1,6 @@
 package ai.basic.x1.adapter.dto;
 
+import ai.basic.x1.entity.enums.ItemTypeEnum;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
@@ -21,6 +22,9 @@ public class DataAnnotateDTO {
     @NotNull(message = "not allow null")
     private Long datasetId;
 
+    @NotNull(message = "not allow null")
+    private ItemTypeEnum operateItemType;
+
     @NotEmpty(message = "not allow null")
     private List<Long> dataIds;
 

backend/src/main/java/ai/basic/x1/adapter/dto/DataAnnotationRecordDTO.java

@@ -1,5 +1,6 @@
 package ai.basic.x1.adapter.dto;
 
+import ai.basic.x1.entity.enums.ItemTypeEnum;
 import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
 import lombok.AllArgsConstructor;
@@ -23,6 +24,8 @@ public class DataAnnotationRecordDTO {
      */
     private Long datasetId;
 
+    private ItemTypeEnum itemType;
+
     /**
      * Serial number
      */

backend/src/main/java/ai/basic/x1/adapter/dto/DataEditDTO.java

@@ -27,6 +27,11 @@ public class DataEditDTO {
     */
    private Long  annotationRecordId;
 
+   /**
+    * Scene id
+    */
+   private Long sceneId;
+
    /**
     * Data id
     */

backend/src/main/java/ai/basic/x1/adapter/dto/DataInfoDTO.java

@@ -2,6 +2,7 @@
 
 import ai.basic.x1.entity.enums.DataAnnotationStatusEnum;
 import ai.basic.x1.entity.enums.DataStatusEnum;
+import ai.basic.x1.entity.enums.ItemTypeEnum;
 import ai.basic.x1.entity.enums.SplitTypeEnum;
 import cn.hutool.core.collection.CollectionUtil;
 import cn.hutool.core.util.ObjectUtil;
@@ -42,11 +43,26 @@ public class DataInfoDTO {
      */
     private String name;
 
+    /**
+     * Sort data name
+     */
+    private String orderName;
+
     /**
      * Content (folder path, version information)
      */
     private List<FileNodeDTO> content;
 
+    /**
+     * Type (indicates continuous frames, non-consecutive frames)
+     */
+    private ItemTypeEnum type;
+
+    /**
+     * Parent ID (Scene ID)
+     */
+    private Long parentId;
+
     /**
      * Data status INVALID,VALID
      */

backend/src/main/java/ai/basic/x1/adapter/dto/DataInfoQueryDTO.java

@@ -32,6 +32,9 @@ public class DataInfoQueryDTO {
     @NotNull(message = "datasetId cannot be null")
     private Long datasetId;
 
+
+    private Long parentId;
+
     /**
      * Data name
      */

backend/src/main/java/ai/basic/x1/adapter/dto/DataModelAnnotateDTO.java

@@ -1,6 +1,7 @@
 package ai.basic.x1.adapter.dto;
 
 import ai.basic.x1.adapter.api.annotation.valid.ValidStringEnum;
+import ai.basic.x1.entity.enums.ItemTypeEnum;
 import ai.basic.x1.entity.enums.ModelCodeEnum;
 import cn.hutool.json.JSONObject;
 import lombok.AllArgsConstructor;
@@ -27,6 +28,8 @@ public class DataModelAnnotateDTO {
     @NotNull(message = "not allow null")
     private Long datasetId;
 
+    private ItemTypeEnum operateItemType;
+
     /**
      * Data id collection
      */

backend/src/main/java/ai/basic/x1/adapter/dto/request/ModelClassReqDTO.java

@@ -15,7 +15,7 @@
 public class ModelClassReqDTO {
 
     /**
-     * 模型id
+     * Model id
      */
     @NotNull(message = "Model id cannot be null")
     private Long modelId;

backend/src/main/java/ai/basic/x1/adapter/dto/response/DataAnnotationResultDTO.java

@@ -1,4 +1,5 @@
 package ai.basic.x1.adapter.dto.response;
+
 import ai.basic.x1.adapter.dto.DataAnnotationClassificationDTO;
 import ai.basic.x1.adapter.dto.DataAnnotationObjectDTO;
 import lombok.AllArgsConstructor;
@@ -18,20 +19,20 @@
 @AllArgsConstructor
 public class DataAnnotationResultDTO {
 
-   /**
-    * 数据ID
-    */
-   private Long  dataId;
+    /**
+     * Data id
+     */
+    private Long dataId;
 
-   /**
-    * Classification 属性取值
-    */
-   private List<DataAnnotationClassificationDTO> classificationValues;
+    /**
+     * Classification values
+     */
+    private List<DataAnnotationClassificationDTO> classificationValues;
 
-   /**
-    * Data 包含的 Object 列表
-    */
-   private List<DataAnnotationObjectDTO>  objects;
+    /**
+     * Object list
+     */
+    private List<DataAnnotationObjectDTO> objects;
 
 
 }

backend/src/main/java/ai/basic/x1/adapter/port/dao/mybatis/mapper/DataAnnotationObjectMapper.java

@@ -21,4 +21,6 @@ public interface DataAnnotationObjectMapper extends ExtendBaseMapper<DataAnnotat
    List<DataAnnotationObject> listByScenario(@Param("scenarioQuery") ScenarioQuery scenarioQuery);
 
    Page<DataAnnotationObject> findDataIdByScenarioPage(Page<DataAnnotationObject> page, @Param("scenarioQuery") ScenarioQuery scenarioQuery);
+
+   List<Long> findDataIdByScenario(@Param("scenarioQuery") ScenarioQuery scenarioQuery);
 }

backend/src/main/java/ai/basic/x1/adapter/port/dao/mybatis/mapper/DataInfoMapper.java

@@ -78,6 +78,9 @@ Long findModelRunDataCount(@Param(Constants.WRAPPER) Wrapper<DataInfo> queryWrap
      */
     Page<DataInfo> selectDataPage(Page<DataInfo> page, @Param(Constants.WRAPPER) Wrapper<DataInfo> queryWrapper, @Param("dataInfoQuery") DataInfoQuery dataInfoQuery);
 
+
+    List<DataInfo> getExportData(@Param(Constants.WRAPPER) Wrapper<DataInfo> queryWrapper, @Param("dataInfoQuery") DataInfoQuery dataInfoQuery);
+
     /**
      * This method overrides the deleteById method of baseMapper
      *
@@ -92,4 +95,14 @@ Long findModelRunDataCount(@Param(Constants.WRAPPER) Wrapper<DataInfo> queryWrap
      * @param datasetId Dataset id
      */
     int deleteByDatasetId(@Param("datasetId") Long datasetId);
+
+
+    /**
+     * Query the first data ID of consecutive frames
+     *
+     * @param datasetId Dataset Id
+     * @param sceneIds  Continuous frame ID
+     * @return
+     */
+    List<Long> selectFirstDataIdBySceneIds(@Param("datasetId") Long datasetId, @Param("sceneIds") List<Long> sceneIds);
 }