Merge pull request #2 from zweihander/feat/href-export

Feat/href export

Author: ernado

_schema/121.json

@@ -9,9 +9,10 @@ import (
 
 // Constructor represents constructor documentation.
 type Constructor struct {
-	Name        string            `json:"name"`
-	Description []string          `json:"description,omitempty"`
-	Fields      map[string]string `json:"fields,omitempty"`
+	Name        string                      `json:"name"`
+	Description []string                    `json:"description,omitempty"`
+	Links       []string                    `json:"links,omitempty"`
+	Fields      map[string]ParamDescription `json:"fields,omitempty"`
 }
 
 // ParseConstructor parses html documentation from reader and produces Constructor.
@@ -20,9 +21,12 @@ func ParseConstructor(reader io.Reader) (*Constructor, error) {
 	if err != nil {
 		return nil, errors.Errorf("failed to parse document: %w", err)
 	}
+
+	desc, links := docDescription(doc)
 	return &Constructor{
 		Name:        docTitle(doc),
-		Description: docDescription(doc),
+		Description: desc,
+		Links:       links,
 		Fields:      docParams(doc),
 	}, nil
 }

constructor.go

@@ -23,13 +23,28 @@ func TestConstructor(t *testing.T) {
 	expected := &Constructor{
 		Name:        "userProfilePhoto",
 		Description: []string{"User profile photo."},
-		Fields: map[string]string{
-			"dc_id":       "DC ID where the photo is stored",
-			"flags":       "Flags, see TL conditional fields",
-			"has_video":   "Whether an animated profile picture is available for this user",
-			"photo_big":   "Location of the file, corresponding to the big profile photo thumbnail",
-			"photo_id":    "Identifier of the respective photoParameter added in Layer 2",
-			"photo_small": "Location of the file, corresponding to the small profile photo thumbnail",
+		Fields: map[string]ParamDescription{
+			"dc_id": {
+				Description: "DC ID where the photo is stored",
+			},
+			"flags": {
+				Description: "Flags, see TL conditional fields¹",
+				Links:       []string{"https://core.telegram.org/mtproto/TL-combinators#conditional-fields"},
+			},
+			"has_video": {
+				Description: "Whether an animated profile picture¹ is available for this user",
+				Links:       []string{"https://core.telegram.org/api/files#animated-profile-pictures"},
+			},
+			"photo_big": {
+				Description: "Location of the file, corresponding to the big profile photo thumbnail",
+			},
+			"photo_id": {
+				Description: "Identifier of the respective photoParameter added in Layer 2¹",
+				Links:       []string{"https://core.telegram.org/api/layers#layer-2"},
+			},
+			"photo_small": {
+				Description: "Location of the file, corresponding to the small profile photo thumbnail",
+			},
 		},
 	}
 	require.Equal(t, expected, v)

constructor_test.go

@@ -6,6 +6,8 @@ import (
 	"strings"
 
 	"github.com/PuerkitoBio/goquery"
+
+	"github.com/gotd/getdoc/href"
 )
 
 // Doc represents full documentation description.
@@ -23,25 +25,27 @@ func docTitle(doc *goquery.Document) string {
 }
 
 // docDescription extracts description lines from document.
-func docDescription(doc *goquery.Document) []string {
-	var description []string
+func docDescription(doc *goquery.Document) (desc, links []string) {
 	doc.Find("#dev_page_content").Each(func(i int, s *goquery.Selection) {
 		s.Children().EachWithBreak(func(i int, selection *goquery.Selection) bool {
 			if selection.Is("p") && selection.Text() != "" {
-				// Trimming space and handling newlines.
+				hrefs := href.Replace(selection)
+
 				text := strings.TrimSpace(selection.Text())
 				for _, part := range strings.Split(text, "\n") {
 					part = strings.TrimSpace(part)
 					if part == "" {
 						continue
 					}
-					description = append(description, part)
+					desc = append(desc, part)
 				}
+
+				links = append(links, addHost(hrefs)...)
 			}
 			return !selection.HasClass("clearfix")
 		})
 	})
-	return description
+	return
 }
 
 // docTableAfter extracts table after selector "after".
@@ -69,19 +73,30 @@ func docTableAfter(doc *goquery.Document, after string) *goquery.Selection {
 	return table.First().Find("tbody > tr")
 }
 
+type ParamDescription struct {
+	Description string   `json:"description"`
+	Links       []string `json:"links"`
+}
+
 // docParams extract parameters documentation from document.
 //
-// Key is parameter name, value is documentation string.
-func docParams(doc *goquery.Document) map[string]string {
-	fields := make(map[string]string)
+// Key is parameter name, value is documentation struct.
+func docParams(doc *goquery.Document) map[string]ParamDescription {
+	fields := make(map[string]ParamDescription)
+
 	docTableAfter(doc, "#parameters").
 		Each(func(i int, row *goquery.Selection) {
 			var rowContents []string
+			var links []string
 			row.Find("td").Each(func(i int, column *goquery.Selection) {
-				rowContents = append(rowContents, strings.TrimSpace(column.Text()))
+				links = addHost(href.Replace(column))
+				rowContents = append(rowContents, column.Text())
 			})
 			if len(rowContents) == 3 {
-				fields[rowContents[0]] = rowContents[2]
+				fields[rowContents[0]] = ParamDescription{
+					Description: rowContents[2],
+					Links:       links,
+				}
 			}
 		})
 	return fields

getdoc.go

@@ -0,0 +1,36 @@
+package href
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestHref(t *testing.T) {
+	tests := []struct {
+		Input string
+		Text  string
+		HREFs []string
+	}{
+		{
+			Input: `<p>Hello! <a href="https://foo.com/bar">Click me!</a></p>`,
+			Text:  `Hello! Click me!¹`,
+			HREFs: []string{"https://foo.com/bar"},
+		},
+		{
+			Input: `<p>Hello! <a href="/foo">Click me</a>   again!</p>`,
+			Text:  `Hello! Click me¹   again!`,
+			HREFs: []string{"/foo"},
+		},
+	}
+
+	for _, test := range tests {
+		doc, err := goquery.NewDocumentFromReader(strings.NewReader(test.Input))
+		assert.NoError(t, err)
+		hrefs := Replace(doc.Selection)
+		assert.Equal(t, test.Text, doc.Text())
+		assert.Equal(t, hrefs, test.HREFs)
+	}
+}

href/href_test.go

@@ -0,0 +1,54 @@
+package href
+
+import (
+	"unicode"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+// Replace replaces all found HREFs with [index] symbol at end of node text.
+// It returns slice of replaced HREFs.
+func Replace(selection *goquery.Selection) (hrefs []string) {
+	replaceHrefsRecursively(selection, map[int]struct{}{}, &hrefs)
+	return hrefs
+}
+
+func replaceHrefsRecursively(selection *goquery.Selection, accum map[int]struct{}, hrefs *[]string) {
+	if _, processed := accum[selection.Index()]; processed {
+		return
+	}
+
+	if path, ok := selection.Attr("href"); ok {
+		accum[selection.Index()] = struct{}{}
+
+		*hrefs = append(*hrefs, path)
+
+		text, cut := cutRightSpaces(selection.Text())
+		text += superscript(len(*hrefs))
+		text += cut
+
+		selection.SetText(text)
+	}
+
+	selection.Find("*").Each(func(i int, s *goquery.Selection) {
+		replaceHrefsRecursively(s, accum, hrefs)
+	})
+}
+
+func cutRightSpaces(input string) (result, cut string) {
+	var (
+		r = []rune(input)
+		c []rune
+	)
+
+	for i := len(r) - 1; i >= 0; i-- {
+		if unicode.IsSpace(r[i]) {
+			c = append(c, r[i])
+			r = r[:i]
+		} else {
+			break
+		}
+	}
+
+	return string(r), string(c)
+}

href/replace.go

@@ -0,0 +1,38 @@
+package href
+
+import "strconv"
+
+func superscript(i int) string {
+	var out []rune
+	for _, r := range strconv.Itoa(i) {
+		var superscript rune
+		switch r {
+		case '0':
+			superscript = '⁰'
+		case '1':
+			superscript = '¹'
+		case '2':
+			superscript = '²'
+		case '3':
+			superscript = '³'
+		case '4':
+			superscript = '⁴'
+		case '5':
+			superscript = '⁵'
+		case '6':
+			superscript = '⁶'
+		case '7':
+			superscript = '⁷'
+		case '8':
+			superscript = '⁸'
+		case '9':
+			superscript = '⁹'
+		default:
+			panic(r)
+		}
+
+		out = append(out, superscript)
+	}
+
+	return string(out)
+}

href/superscript.go

@@ -0,0 +1,22 @@
+package getdoc
+
+import (
+	"net/url"
+)
+
+func addHost(hrefs []string) (s []string) {
+	for _, href := range hrefs {
+		u, err := url.Parse(href)
+		if err != nil {
+			panic(err)
+		}
+
+		if u.Host == "" {
+			href = "https://core.telegram.org" + href
+		}
+
+		s = append(s, href)
+	}
+
+	return
+}

internal/bindata.go

@@ -10,10 +10,11 @@ import (
 
 // Method represents method documentation.
 type Method struct {
-	Name        string            `json:"name"`
-	Description []string          `json:"description,omitempty"`
-	Parameters  map[string]string `json:"parameters,omitempty"`
-	Errors      []Error           `json:"errors,omitempty"`
+	Name        string                      `json:"name"`
+	Description []string                    `json:"description,omitempty"`
+	Links       []string                    `json:"links,omitempty"`
+	Parameters  map[string]ParamDescription `json:"parameters,omitempty"`
+	Errors      []Error                     `json:"errors,omitempty"`
 }
 
 // Error represent possible error documentation.
@@ -55,9 +56,12 @@ func ParseMethod(reader io.Reader) (*Method, error) {
 	if err != nil {
 		return nil, errors.Errorf("failed to parse document: %w", err)
 	}
+
+	desc, links := docDescription(doc)
 	return &Method{
 		Name:        docTitle(doc),
-		Description: docDescription(doc),
+		Description: desc,
+		Links:       links,
 		Parameters:  docParams(doc),
 		Errors:      docErrors(doc),
 	}, nil

links.go

@@ -23,10 +23,10 @@ func TestParseMethod(t *testing.T) {
 	expected := &Method{
 		Name:        "langpack.getDifference",
 		Description: []string{"Get new strings in languagepack"},
-		Parameters: map[string]string{
-			"from_version": "Previous localization pack version",
-			"lang_code":    "Language code",
-			"lang_pack":    "Language pack",
+		Parameters: map[string]ParamDescription{
+			"from_version": {Description: "Previous localization pack version"},
+			"lang_code":    {Description: "Language code"},
+			"lang_pack":    {Description: "Language pack"},
 		},
 		Errors: []Error{
 			{Code: 400, Type: "LANG_PACK_INVALID", Description: "The provided language pack is invalid"},

method.go

@@ -10,7 +10,8 @@ import (
 // Type represents type (aka class) documentation.
 type Type struct {
 	Name        string   `json:"name"`
-	Description []string `json:"description"`
+	Description []string `json:"description,omitempty"`
+	Links       []string `json:"links,omitempty"`
 }
 
 // ParseType parses Type documentation from reader.
@@ -19,8 +20,11 @@ func ParseType(reader io.Reader) (*Type, error) {
 	if err != nil {
 		return nil, errors.Errorf("failed to parse document: %w", err)
 	}
+
+	desc, links := docDescription(doc)
 	return &Type{
 		Name:        docTitle(doc),
-		Description: docDescription(doc),
+		Description: desc,
+		Links:       links,
 	}, nil
 }