Skip to content

Commit 80aca4c

Browse files
DevAloneDevAlone
committed
new orm works, some docs
1 parent 4cdae55 commit 80aca4c

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

58 files changed

+16510
-248
lines changed

‎.gitignore

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,4 @@ __pycache__
55
db.sqlite3*
66
*.swp
77
proxy_py/settings.py
8-
migrations
9-
alembic.ini
10-
alembic
118
env/
12-
local/

‎CHANGELOG

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# CHANGELOG
2+
3+
## v1.0.2
4+
5+
### Improved proxy checking
6+
7+
Multiple sites for checking proxies. New information about proxies(real IP, country, region, city)
8+
9+
10+
## v1.0.1
11+
12+
### Fix server API
13+
14+
Now it's sending you status code 400/500 if error happened.
15+
16+
17+
## v1.0
18+
19+
first release

‎TODO

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
new web interface
2+
statistics for proxies and for collectors
3+
logging

‎collectors/collector.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
# TODO: add wrapper for doing requests and saving its cookies and UserAgent
22

3+
34
class AbstractCollector:
45
# this method should return proxies in any of the following formats:
56
# ip:port
@@ -13,7 +14,7 @@ async def load_state(self, state):
1314
self.last_processing_time = state.last_processing_time
1415
self.processing_period = state.processing_period
1516

16-
async def save_state(self, state):
17+
async def set_state(self, state):
1718
state.last_processing_time = self.last_processing_time
1819
state.processing_period = self.processing_period
1920

‎collectors_list.py

Lines changed: 29 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
import asyncio
2+
3+
import peewee
4+
15
from models import db, CollectorState
26
from proxy_py import settings
37

@@ -28,31 +32,36 @@
2832

2933

3034
# init db
31-
# TODO: do it
32-
# for module_name, CollectorType in collectors.items():
33-
# collectorState = session.query(CollectorState).filter(CollectorState.identifier == module_name).first()
34-
#
35-
# if not collectorState:
36-
# session.add(CollectorState(
37-
# identifier=module_name,
38-
# processing_period=CollectorType.processing_period,
39-
# last_processing_time=0,
40-
# ))
41-
# session.commit()
42-
43-
44-
# def get_collector_state(module_name : str):
45-
# collectorState = session.query(CollectorState).filter(CollectorState.identifier == module_name).first()
46-
# if not collectorState:
47-
# raise CollectorNotFoundException()
48-
#
49-
# return collectorState
35+
36+
for module_name, CollectorType in collectors.items():
37+
try:
38+
collector_state = asyncio.get_event_loop().run_until_complete(db.get(
39+
CollectorState.select().where(CollectorState.identifier == module_name)
40+
))
41+
except CollectorState.DoesNotExist:
42+
asyncio.get_event_loop().run_until_complete(db.create(
43+
CollectorState,
44+
identifier=module_name,
45+
processing_period=CollectorType.processing_period,
46+
last_processing_time=0,
47+
))
48+
49+
50+
def get_collector_state(module_name: str):
51+
try:
52+
collector_state = asyncio.get_event_loop().run_until_complete(db.get(
53+
CollectorState.select().where(CollectorState.identifier == module_name)
54+
))
55+
except CollectorState.DoesNotExist:
56+
raise CollectorNotFoundException()
57+
58+
return collector_state
5059

5160

5261
def get_collector_of_module_name(module_name: str):
5362
if module_name not in collectors:
5463
raise CollectorNotFoundException(
55-
'Probably some collector exists in database but not in filesystem. module_name = {}'.format(module_name)
64+
"Probably some collector exists in database but not in filesystem. module_name = {}".format(module_name)
5665
)
5766

5867
return collectors[module_name]

‎docs/Makefile

Lines changed: 216 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,216 @@
1+
# Makefile for Sphinx documentation
2+
#
3+
4+
# You can set these variables from the command line.
5+
SPHINXOPTS =
6+
SPHINXBUILD = sphinx-build
7+
PAPER =
8+
BUILDDIR = _build
9+
10+
# User-friendly check for sphinx-build
11+
ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
12+
$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
13+
endif
14+
15+
# Internal variables.
16+
PAPEROPT_a4 = -D latex_paper_size=a4
17+
PAPEROPT_letter = -D latex_paper_size=letter
18+
ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
19+
# the i18n builder cannot share the environment and doctrees with the others
20+
I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
21+
22+
.PHONY: help
23+
help:
24+
@echo "Please use \`make <target>' where <target> is one of"
25+
@echo " html to make standalone HTML files"
26+
@echo " dirhtml to make HTML files named index.html in directories"
27+
@echo " singlehtml to make a single large HTML file"
28+
@echo " pickle to make pickle files"
29+
@echo " json to make JSON files"
30+
@echo " htmlhelp to make HTML files and a HTML help project"
31+
@echo " qthelp to make HTML files and a qthelp project"
32+
@echo " applehelp to make an Apple Help Book"
33+
@echo " devhelp to make HTML files and a Devhelp project"
34+
@echo " epub to make an epub"
35+
@echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
36+
@echo " latexpdf to make LaTeX files and run them through pdflatex"
37+
@echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx"
38+
@echo " text to make text files"
39+
@echo " man to make manual pages"
40+
@echo " texinfo to make Texinfo files"
41+
@echo " info to make Texinfo files and run them through makeinfo"
42+
@echo " gettext to make PO message catalogs"
43+
@echo " changes to make an overview of all changed/added/deprecated items"
44+
@echo " xml to make Docutils-native XML files"
45+
@echo " pseudoxml to make pseudoxml-XML files for display purposes"
46+
@echo " linkcheck to check all external links for integrity"
47+
@echo " doctest to run all doctests embedded in the documentation (if enabled)"
48+
@echo " coverage to run coverage check of the documentation (if enabled)"
49+
50+
.PHONY: clean
51+
clean:
52+
rm -rf $(BUILDDIR)/*
53+
54+
.PHONY: html
55+
html:
56+
$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
57+
@echo
58+
@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
59+
60+
.PHONY: dirhtml
61+
dirhtml:
62+
$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
63+
@echo
64+
@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
65+
66+
.PHONY: singlehtml
67+
singlehtml:
68+
$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
69+
@echo
70+
@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
71+
72+
.PHONY: pickle
73+
pickle:
74+
$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
75+
@echo
76+
@echo "Build finished; now you can process the pickle files."
77+
78+
.PHONY: json
79+
json:
80+
$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
81+
@echo
82+
@echo "Build finished; now you can process the JSON files."
83+
84+
.PHONY: htmlhelp
85+
htmlhelp:
86+
$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
87+
@echo
88+
@echo "Build finished; now you can run HTML Help Workshop with the" \
89+
".hhp project file in $(BUILDDIR)/htmlhelp."
90+
91+
.PHONY: qthelp
92+
qthelp:
93+
$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
94+
@echo
95+
@echo "Build finished; now you can run "qcollectiongenerator" with the" \
96+
".qhcp project file in $(BUILDDIR)/qthelp, like this:"
97+
@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/proxy_py.qhcp"
98+
@echo "To view the help file:"
99+
@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/proxy_py.qhc"
100+
101+
.PHONY: applehelp
102+
applehelp:
103+
$(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp
104+
@echo
105+
@echo "Build finished. The help book is in $(BUILDDIR)/applehelp."
106+
@echo "N.B. You won't be able to view it unless you put it in" \
107+
"~/Library/Documentation/Help or install it in your application" \
108+
"bundle."
109+
110+
.PHONY: devhelp
111+
devhelp:
112+
$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
113+
@echo
114+
@echo "Build finished."
115+
@echo "To view the help file:"
116+
@echo "# mkdir -p $$HOME/.local/share/devhelp/proxy_py"
117+
@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/proxy_py"
118+
@echo "# devhelp"
119+
120+
.PHONY: epub
121+
epub:
122+
$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
123+
@echo
124+
@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
125+
126+
.PHONY: latex
127+
latex:
128+
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
129+
@echo
130+
@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
131+
@echo "Run \`make' in that directory to run these through (pdf)latex" \
132+
"(use \`make latexpdf' here to do that automatically)."
133+
134+
.PHONY: latexpdf
135+
latexpdf:
136+
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
137+
@echo "Running LaTeX files through pdflatex..."
138+
$(MAKE) -C $(BUILDDIR)/latex all-pdf
139+
@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
140+
141+
.PHONY: latexpdfja
142+
latexpdfja:
143+
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
144+
@echo "Running LaTeX files through platex and dvipdfmx..."
145+
$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
146+
@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
147+
148+
.PHONY: text
149+
text:
150+
$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
151+
@echo
152+
@echo "Build finished. The text files are in $(BUILDDIR)/text."
153+
154+
.PHONY: man
155+
man:
156+
$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
157+
@echo
158+
@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
159+
160+
.PHONY: texinfo
161+
texinfo:
162+
$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
163+
@echo
164+
@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
165+
@echo "Run \`make' in that directory to run these through makeinfo" \
166+
"(use \`make info' here to do that automatically)."
167+
168+
.PHONY: info
169+
info:
170+
$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
171+
@echo "Running Texinfo files through makeinfo..."
172+
make -C $(BUILDDIR)/texinfo info
173+
@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
174+
175+
.PHONY: gettext
176+
gettext:
177+
$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
178+
@echo
179+
@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
180+
181+
.PHONY: changes
182+
changes:
183+
$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
184+
@echo
185+
@echo "The overview file is in $(BUILDDIR)/changes."
186+
187+
.PHONY: linkcheck
188+
linkcheck:
189+
$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
190+
@echo
191+
@echo "Link check complete; look for any errors in the above output " \
192+
"or in $(BUILDDIR)/linkcheck/output.txt."
193+
194+
.PHONY: doctest
195+
doctest:
196+
$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
197+
@echo "Testing of doctests in the sources finished, look at the " \
198+
"results in $(BUILDDIR)/doctest/output.txt."
199+
200+
.PHONY: coverage
201+
coverage:
202+
$(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage
203+
@echo "Testing of coverage in the sources finished, look at the " \
204+
"results in $(BUILDDIR)/coverage/python.txt."
205+
206+
.PHONY: xml
207+
xml:
208+
$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
209+
@echo
210+
@echo "Build finished. The XML files are in $(BUILDDIR)/xml."
211+
212+
.PHONY: pseudoxml
213+
pseudoxml:
214+
$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
215+
@echo
216+
@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."

‎docs/_build/doctrees/api/v1/intro.doctree

2.93 KB
Binary file not shown.

‎docs/_build/doctrees/api/v1/overview.doctree

16.3 KB
Binary file not shown.

‎docs/_build/doctrees/environment.pickle

5.44 KB
Binary file not shown.

‎docs/_build/doctrees/index.doctree

4.96 KB
Binary file not shown.

0 commit comments

Comments
 (0)