Merge pull request #1021 from Starnop/supernode-priority

feature: add weight for node config

Author: allencloud

cmd/dfget/app/root.go

@@ -72,7 +72,10 @@ func init() {
 // runDfget does some init operations and starts to download.
 func runDfget() error {
 	// get config from property files
-	propResults := initProperties()
+	propResults, err := initProperties()
+	if err != nil {
+		return err
+	}
 
 	// initialize logger
 	if err := initClientLog(); err != nil {
@@ -91,10 +94,6 @@ func runDfget() error {
 
 	cfg.Filter = transFilter(filter)
 
-	if err := handleNodes(); err != nil {
-		return err
-	}
-
 	if err := checkParameters(); err != nil {
 		return err
 	}
@@ -106,10 +105,10 @@ func runDfget() error {
 	logrus.Infof("get init config:%v", cfg)
 
 	// enter the core process
-	err := core.Start(cfg)
-	printer.Println(resultMsg(cfg, time.Now(), err))
-	if err != nil {
-		os.Exit(err.Code)
+	dfError := core.Start(cfg)
+	printer.Println(resultMsg(cfg, time.Now(), dfError))
+	if dfError != nil {
+		os.Exit(dfError.Code)
 	}
 	return nil
 }
@@ -122,12 +121,12 @@ func checkParameters() error {
 }
 
 // initProperties loads config from property files.
-func initProperties() []*propertiesResult {
+func initProperties() ([]*propertiesResult, error) {
 	var results []*propertiesResult
 	properties := config.NewProperties()
 	for _, v := range cfg.ConfigFiles {
-		var err error
-		if err = properties.Load(v); err == nil {
+		err := properties.Load(v)
+		if err == nil {
 			break
 		}
 		results = append(results, &propertiesResult{
@@ -137,8 +136,12 @@ func initProperties() []*propertiesResult {
 		})
 	}
 
-	if cfg.Nodes == nil {
-		cfg.Nodes = properties.Nodes
+	supernodes := cfg.Supernodes
+	if supernodes == nil {
+		supernodes = properties.Supernodes
+	}
+	if supernodes != nil {
+		cfg.Nodes = config.NodeWightSlice2StringSlice(supernodes)
 	}
 
 	if cfg.LocalLimit == 0 {
@@ -173,7 +176,7 @@ func initProperties() []*propertiesResult {
 	cfg.RV.SystemDataDir = path.Join(cfg.WorkHome, "data")
 	cfg.RV.FileLength = -1
 
-	return results
+	return results, nil
 }
 
 // initClientLog initializes dfget client's logger.
@@ -234,8 +237,8 @@ func initFlags() {
 			"\nin this way, different but actually the same URLs can reuse the same downloading task")
 	flagSet.StringSliceVar(&cfg.Header, "header", nil,
 		"http header, eg: --header='Accept: *' --header='Host: abc'")
-	flagSet.StringSliceVarP(&cfg.Nodes, "node", "n", nil,
-		"specify the addresses(host:port) of supernodes")
+	flagSet.VarP(config.NewSupernodesValue(&cfg.Supernodes, nil), "node", "n",
+		"specify the addresses(host:port=weight) of supernodes where the host is necessary, the port(default: 8002) and the weight(default:1) are optional. And the type of weight must be integer")
 	flagSet.BoolVar(&cfg.Notbs, "notbs", false,
 		"disable back source downloading for requested file when p2p fails to download it")
 	flagSet.BoolVar(&cfg.DFDaemon, "dfdaemon", false,
@@ -275,21 +278,6 @@ func transFilter(filter string) []string {
 	return strings.Split(filter, "&")
 }
 
-func handleNodes() error {
-	nodes := make([]string, 0)
-
-	for _, v := range cfg.Nodes {
-		// TODO: check the validity of v.
-		if strings.IndexByte(v, ':') > 0 {
-			nodes = append(nodes, v)
-			continue
-		}
-		nodes = append(nodes, fmt.Sprintf("%s:%d", v, config.DefaultSupernodePort))
-	}
-	cfg.Nodes = nodes
-	return nil
-}
-
 func resultMsg(cfg *config.Config, end time.Time, e *errortypes.DfError) string {
 	if e != nil {
 		return fmt.Sprintf("download FAIL(%d) cost:%.3fs length:%d reason:%d error:%v",

cmd/dfget/app/root_test.go

@@ -37,7 +37,7 @@ type dfgetSuit struct {
 
 func (suit *dfgetSuit) Test_initFlagsNoArguments() {
 	initProperties()
-	suit.Equal(cfg.Nodes, []string{"127.0.0.1"})
+	suit.Equal(cfg.Nodes, []string{"127.0.0.1:8002"})
 	suit.Equal(cfg.LocalLimit, 20*rate.MB)
 	suit.Equal(cfg.TotalLimit, 20*rate.MB)
 	suit.Equal(cfg.Notbs, false)
@@ -69,11 +69,11 @@ func (suit *dfgetSuit) Test_initProperties() {
 		{configs: nil,
 			expected: config.NewProperties()},
 		{configs: []string{iniFile, yamlFile},
-			expected: newProp(0, 0, 0, "1.1.1.1")},
+			expected: newProp(0, 0, 0, "1.1.1.1:8002")},
 		{configs: []string{yamlFile, iniFile},
-			expected: newProp(int(rate.KB*1000), int(rate.KB*1000), 0, "1.1.1.2")},
+			expected: newProp(int(rate.KB*1000), int(rate.KB*1000), 0, "1.1.1.2:8002")},
 		{configs: []string{filepath.Join(dirName, "x"), yamlFile},
-			expected: newProp(int(rate.KB*1000), int(rate.KB*1000), 0, "1.1.1.2")},
+			expected: newProp(int(rate.KB*1000), int(rate.KB*1000), 0, "1.1.1.2:8002")},
 	}
 
 	for _, v := range cases {
@@ -84,7 +84,7 @@ func (suit *dfgetSuit) Test_initProperties() {
 			"--locallimit", v.expected.LocalLimit.String(),
 			"--totallimit", v.expected.TotalLimit.String()})
 		initProperties()
-		suit.EqualValues(cfg.Nodes, v.expected.Nodes)
+		suit.EqualValues(cfg.Nodes, config.NodeWightSlice2StringSlice(v.expected.Supernodes))
 		suit.Equal(cfg.LocalLimit, v.expected.LocalLimit)
 		suit.Equal(cfg.TotalLimit, v.expected.TotalLimit)
 		suit.Equal(cfg.ClientQueueSize, v.expected.ClientQueueSize)
@@ -134,7 +134,7 @@ func TestSuite(t *testing.T) {
 func newProp(local int, total int, size int, nodes ...string) *config.Properties {
 	p := config.NewProperties()
 	if nodes != nil {
-		p.Nodes = nodes
+		p.Supernodes, _ = config.ParseNodesSlice(nodes)
 	}
 	if local != 0 {
 		p.LocalLimit = rate.Rate(local)

dfget/config/config.go

@@ -33,7 +33,6 @@ import (
 	"github.com/dragonflyoss/Dragonfly/pkg/printer"
 	"github.com/dragonflyoss/Dragonfly/pkg/rate"
 	"github.com/dragonflyoss/Dragonfly/pkg/stringutils"
-
 	"github.com/pkg/errors"
 	"gopkg.in/gcfg.v1"
 	"gopkg.in/warnings.v0"
@@ -50,14 +49,18 @@ import (
 // Since 0.2.0, the INI config is just to be compatible with previous versions.
 // The YAML config will have more properties:
 // 		nodes:
-// 		    - 127.0.0.1
-// 		    - 10.10.10.1
+// 		    - 127.0.0.1=1
+// 		    - 10.10.10.1:8002=2
 // 		localLimit: 20M
 // 		totalLimit: 20M
 // 		clientQueueSize: 6
 type Properties struct {
-	// Nodes specify supernodes.
-	Nodes []string `yaml:"nodes,omitempty" json:"nodes,omitempty"`
+	// Supernodes specify supernodes with weight.
+	// The type of weight must be integer.
+	// All weights will be divided by the greatest common divisor in the end.
+	//
+	// E.g. ["192.168.33.21=1", "192.168.33.22=2"]
+	Supernodes []*NodeWight `yaml:"nodes,omitempty" json:"nodes,omitempty"`
 
 	// LocalLimit rate limit about a single download task, format: G(B)/g/M(B)/m/K(B)/k/B
 	// pure number will also be parsed as Byte.
@@ -85,7 +88,7 @@ type Properties struct {
 // NewProperties creates a new properties with default values.
 func NewProperties() *Properties {
 	return &Properties{
-		Nodes:           []string{DefaultNode},
+		Supernodes:      GetDefaultSupernodesValue(),
 		LocalLimit:      DefaultLocalLimit,
 		MinRate:         DefaultMinRate,
 		ClientQueueSize: DefaultClientQueueSize,
@@ -123,8 +126,13 @@ func (p *Properties) loadFromIni(path string) error {
 			return fmt.Errorf("read ini config from %s error: %v", path, err)
 		}
 	}
-	p.Nodes = strings.Split(oldConfig.Node.Address, ",")
-	return nil
+
+	nodes, err := ParseNodesString(oldConfig.Node.Address)
+	if err != nil {
+		return errors.Wrapf(err, "failed to handle nodes")
+	}
+	p.Supernodes = nodes
+	return err
 }
 
 func (p *Properties) fileType(path string) string {
@@ -197,6 +205,9 @@ type Config struct {
 	// If set true, log level will be 'debug'.
 	Verbose bool `json:"verbose,omitempty"`
 
+	// Nodes specify supernodes.
+	Nodes []string `json:"-"`
+
 	// Start time.
 	StartTime time.Time `json:"-"`
 

dfget/config/config_test.go

@@ -173,19 +173,31 @@ func (suite *ConfigSuite) TestProperties_Load(c *check.C) {
 			content: "nodes:\n\t- 10.10.10.1", errMsg: "yaml", expected: nil},
 		{create: true, ext: "yaml",
 			content: "nodes:\n  - 10.10.10.1\n  - 10.10.10.2\n",
-			errMsg:  "", expected: &Properties{Nodes: []string{"10.10.10.1", "10.10.10.2"}}},
+			errMsg:  "", expected: &Properties{Supernodes: []*NodeWight{
+				{"10.10.10.1:8002", 1},
+				{"10.10.10.2:8002", 1},
+			}}},
 		{create: true, ext: "yaml",
 			content: "totalLimit: 10M",
 			errMsg:  "", expected: &Properties{TotalLimit: 10 * rate.MB}},
 		{create: false, ext: "ini", content: "[node]\naddress=1.1.1.1", errMsg: "read ini config"},
 		{create: true, ext: "ini", content: "[node]\naddress=1.1.1.1",
-			expected: &Properties{Nodes: []string{"1.1.1.1"}}},
+			expected: &Properties{Supernodes: []*NodeWight{
+				{"1.1.1.1:8002", 1},
+			}}},
 		{create: true, ext: "conf", content: "[node]\naddress=1.1.1.1",
-			expected: &Properties{Nodes: []string{"1.1.1.1"}}},
+			expected: &Properties{Supernodes: []*NodeWight{
+				{"1.1.1.1:8002", 1},
+			}}},
 		{create: true, ext: "conf", content: "[node]\naddress=1.1.1.1,1.1.1.2",
-			expected: &Properties{Nodes: []string{"1.1.1.1", "1.1.1.2"}}},
+			expected: &Properties{Supernodes: []*NodeWight{
+				{"1.1.1.1:8002", 1},
+				{"1.1.1.2:8002", 1},
+			}}},
 		{create: true, ext: "conf", content: "[node]\naddress=1.1.1.1\n[totalLimit]",
-			expected: &Properties{Nodes: []string{"1.1.1.1"}}},
+			expected: &Properties{Supernodes: []*NodeWight{
+				{"1.1.1.1:8002", 1},
+			}}},
 	}
 
 	for idx, v := range cases {

dfget/config/constants.go

@@ -54,6 +54,7 @@ const (
 	DefaultMinRate         = 64 * rate.KB
 	DefaultTotalLimit      = 20 * rate.MB
 	DefaultClientQueueSize = 6
+	DefaultSupernodeWeight = 1
 )
 
 /* http headers */