sample Url:
site.com/category/%D9%81%D8%AA%D9%88%DA%AF%D8%B1%D8%A7%D9%81%DB%8C_%D8%B1%DB%8C%D9%86%D9%88%D9%BE%D9%84%D8%A7%D8%B3%D8%AA%DB%8C/
url config:
url(r'^category/(?P<page_slug>.*)/$', views.category, name='category'),
passenger config:
import imp
import os
import sys
sys.path.insert(0, os.path.dirname(__file__))
wsgi = imp.load_source('wsgi', 'photography/wsgi.py')
application = wsgi.application
wsgi config:
"""
WSGI config for photography project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/1.10/howto/deployment/wsgi/
"""
import os
from django.core.wsgi import get_wsgi_application
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "photography.settings")
#os.environ["DJANGO_SETTINGS_MODULE"] = "photography.settings"
application = get_wsgi_application()
but response 404 not found in url!
in problem for all url persian slug.
when change config wsgi to unquote:
from urllib.parse import unquote
def application(environ, start_fn):
environ['PATH_INFO'] = unquote(environ['PATH_INFO'])
app = get_wsgi_application()
print(environ)
return app(environ, start_fn)
change url into:
site.com/category/%C3%99%C2%81%C3%98%C2%AA%C3%99%C2%88%C3%9A%C2%AF%C3%98%C2%B1%C3%98%C2%A7%C3%99/tag/%D9%81%D8%AA%D9%88%DA%AF%D8%B1%D8%A7%D9%81%DB%8C-%D9%BE%D8%B2%D8%B4%DA%A9%DB%8C/
But there is an open problem !
I applied all the changes I found with the search, but there is still a problem!
Output one of the changes in the wsgi:
App 3585081 output: set_script_prefix(get_script_name(environ))
App 3585081 output: File "/home/sepandteb/virtualenv/sepandteb/3.5/lib/python3.5/site-packages/django/core/handlers/wsgi.py", line 210, in get_script_name
App 3585081 output: return script_name.decode(UTF_8)
App 3585081 output: UnicodeDecodeError: 'utf-8' codec can't decode byte 0xc3 in position 27: unexpected end of data
use encode('utf-8') for Persian character and set # -*- coding: utf-8 -*- on top of file.
for example when you save data on DB or read on DB use this function.
i.e :
slug.encode('utf-8')
You can use "uri_to_iri" in the view, for example:
from django.shortcuts import get_object_or_404
from django.utils.encoding import uri_to_iri
def blog_detail(request, slug):
post= get_object_or_404(Post, slug=uri_to_iri(slug))
Related
I'm using Hydra for training machine learning models. It's great for doing complex commands like python train.py data=MNIST batch_size=64 loss=l2. However, if I want to then run the trained model with the same parameters, I have to do something like python reconstruct.py --config_file path_to_previous_job/.hydra/config.yaml. I then use argparse to load in the previous yaml and use the compose API to initialize the Hydra environment. The path to the trained model is inferred from the path to Hydra's .yaml file. If I want to modify one of the parameters, I have to add additional argparse parameters and run something like python reconstruct.py --config_file path_to_previous_job/.hydra/config.yaml --batch_size 128. The code then manually overrides any Hydra parameters with those that were specified on the command line.
What's the right way of doing this?
My current code looks something like the following:
train.py:
import hydra
#hydra.main(config_name="config", config_path="conf")
def main(cfg):
# [training code using cfg.data, cfg.batch_size, cfg.loss etc.]
# [code outputs model checkpoint to job folder generated by Hydra]
main()
reconstruct.py:
import argparse
import os
from hydra.experimental import initialize, compose
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('hydra_config')
parser.add_argument('--batch_size', type=int)
# [other flags and parameters I may need to override]
args = parser.parse_args()
# Create the Hydra environment.
initialize()
cfg = compose(config_name=args.hydra_config)
# Since checkpoints are stored next to the .hydra, we manually generate the path.
checkpoint_dir = os.path.dirname(os.path.dirname(args.hydra_config))
# Manually override any parameters which can be changed on the command line.
batch_size = args.batch_size if args.batch_size else cfg.data.batch_size
# [code which uses checkpoint_dir to load the model]
# [code which uses both batch_size and params in cfg to set up the data etc.]
This is my first time posting, so let me know if I should clarify anything.
If you want to load the previous config as is and not change it, use OmegaConf.load(file_path).
If you want to re-compose the config (and it sounds like you do, because you added that you want override things), I recommend that you use the Compose API and pass in parameters from the overrides file in the job output directory (next to the stored config.yaml), but concatenate the current run parameters.
This script seems to be doing the job:
import os
from dataclasses import dataclass
from os.path import join
from typing import Optional
from omegaconf import OmegaConf
import hydra
from hydra import compose
from hydra.core.config_store import ConfigStore
from hydra.core.hydra_config import HydraConfig
from hydra.utils import to_absolute_path
# You can also use a yaml config file instead of this Structured Config
#dataclass
class Config:
load_checkpoint: Optional[str] = None
batch_size: int = 16
loss: str = "l2"
cs = ConfigStore.instance()
cs.store(name="config", node=Config)
#hydra.main(config_path=".", config_name="config")
def my_app(cfg: Config) -> None:
if cfg.load_checkpoint is not None:
output_dir = to_absolute_path(cfg.load_checkpoint)
original_overrides = OmegaConf.load(join(output_dir, ".hydra/overrides.yaml"))
current_overrides = HydraConfig.get().overrides.task
hydra_config = OmegaConf.load(join(output_dir, ".hydra/hydra.yaml"))
# getting the config name from the previous job.
config_name = hydra_config.hydra.job.config_name
# concatenating the original overrides with the current overrides
overrides = original_overrides + current_overrides
# compose a new config from scratch
cfg = compose(config_name, overrides=overrides)
# train
print("Running in ", os.getcwd())
print(OmegaConf.to_yaml(cfg))
if __name__ == "__main__":
my_app()
~/tmp$ python train.py
Running in /home/omry/tmp/outputs/2021-04-19/21-23-13
load_checkpoint: null
batch_size: 16
loss: l2
~/tmp$ python train.py load_checkpoint=/home/omry/tmp/outputs/2021-04-19/21-23-13
Running in /home/omry/tmp/outputs/2021-04-19/21-23-22
load_checkpoint: /home/omry/tmp/outputs/2021-04-19/21-23-13
batch_size: 16
loss: l2
~/tmp$ python train.py load_checkpoint=/home/omry/tmp/outputs/2021-04-19/21-23-13 batch_size=32
Running in /home/omry/tmp/outputs/2021-04-19/21-23-28
load_checkpoint: /home/omry/tmp/outputs/2021-04-19/21-23-13
batch_size: 32
loss: l2
I recently start web-scraping using scrapy, I generated a list of urls that I want to scrape from into a txt document separate by a new line. This is my crawler code:
import scrapy
import csv
import sys
from realtor.items import RealtorItem
from scrapy.spider import BaseSpider
#from scrapy.selector import HtmlXPathSelector
#from realtor.items import RealtorItem
class RealtorSpider(scrapy.Spider):
name = "realtor"
allowed_domains = ["realtor.com"]
with open('realtor2.txt') as f:
start_urls = [url.strip() for url in f.readlines()]
def parse(self, response):
#hxs = HtmlXPathSelector(response)
#sites = hxs.select('//div/li/div/a/#href')
sites = response.xpath('//a[contains(#href, "/realestateandhomes-detail/")]')
items = []
for site in sites:
print(site.extract())
item = RealtorItem()
item['link'] = site.xpath('#href').extract()
items.append(item)
return items
now my goal is to read the links from realtor2.txt and start parsing through them, however I get a valueError missing scheme in request URL :
File "C:\Users\Ash\Anaconda2\lib\site-packages\scrapy\http\request\__init__.py", line 58, in _set_url
raise ValueError('Missing scheme in request url: %s' % self._url)
ValueError: Missing scheme in request url:
%FF%FEw%00w%00w%00.%00r%00e%00a%00l%00t%00o%00r%00.%00c%00o%00m%00/%00r%00e%00a%00l%00e%00s%00t%00a%00t%00e%00a%00n%00d%00h%00o%00m%00e%00s%00-%00d%00e%00t%00a%00i%00l%00/%005%000%00-%00M%00e%00n%00o%00r%00e%00s%00-%00A%00v%00e%00-%00A%00p%00t%00-%006%001%000%00_%00C%00o%00r%00a%00l%00-%00G%00a%00b%00l%00e%00s%00_%00F%00L%00_%003%003%001%003%004%00_%00M%005%003%008%000%006%00-%005%008%006%007%007%00%0D%00
2017-06-25 22:28:35 [scrapy.core.engine] INFO: Closing spider (finished)
I think there may be an issue while defining start_urls, but I dont know how to proceed,
"ValueError: Missing scheme in request url" means that you are missing http.
You can use urljoin to avoid this problem.
I'm creating a fork of my Plone site (which has not been forked for a long time). This site has a special catalog object for user profiles (a special Archetypes-based object type) which is called portal_user_catalog:
$ bin/instance debug
>>> portal = app.Plone
>>> print [d for d in portal.objectMap() if d['meta_type'] == 'Plone Catalog Tool']
[{'meta_type': 'Plone Catalog Tool', 'id': 'portal_catalog'},
{'meta_type': 'Plone Catalog Tool', 'id': 'portal_user_catalog'}]
This looks reasonable because the user profiles don't have most of the indexes of the "normal" objects, but have a small set of own indexes.
Since I found no way how to create this object from scratch, I exported it from the old site (as portal_user_catalog.zexp) and imported it in the new site. This seemed to work, but I can't add objects to the imported catalog, not even by explicitly calling the catalog_object method. Instead, the user profiles are added to the standard portal_catalog.
Now I found a module in my product which seems to serve the purpose (Products/myproduct/exportimport/catalog.py):
"""Catalog tool setup handlers.
$Id: catalog.py 77004 2007-06-24 08:57:54Z yuppie $
"""
from Products.GenericSetup.utils import exportObjects
from Products.GenericSetup.utils import importObjects
from Products.CMFCore.utils import getToolByName
from zope.component import queryMultiAdapter
from Products.GenericSetup.interfaces import IBody
def importCatalogTool(context):
"""Import catalog tool.
"""
site = context.getSite()
obj = getToolByName(site, 'portal_user_catalog')
parent_path=''
if obj and not obj():
importer = queryMultiAdapter((obj, context), IBody)
path = '%s%s' % (parent_path, obj.getId().replace(' ', '_'))
__traceback_info__ = path
print [importer]
if importer:
print importer.name
if importer.name:
path = '%s%s' % (parent_path, 'usercatalog')
print path
filename = '%s%s' % (path, importer.suffix)
print filename
body = context.readDataFile(filename)
if body is not None:
importer.filename = filename # for error reporting
importer.body = body
if getattr(obj, 'objectValues', False):
for sub in obj.objectValues():
importObjects(sub, path+'/', context)
def exportCatalogTool(context):
"""Export catalog tool.
"""
site = context.getSite()
obj = getToolByName(site, 'portal_user_catalog', None)
if tool is None:
logger = context.getLogger('catalog')
logger.info('Nothing to export.')
return
parent_path=''
exporter = queryMultiAdapter((obj, context), IBody)
path = '%s%s' % (parent_path, obj.getId().replace(' ', '_'))
if exporter:
if exporter.name:
path = '%s%s' % (parent_path, 'usercatalog')
filename = '%s%s' % (path, exporter.suffix)
body = exporter.body
if body is not None:
context.writeDataFile(filename, body, exporter.mime_type)
if getattr(obj, 'objectValues', False):
for sub in obj.objectValues():
exportObjects(sub, path+'/', context)
I tried to use it, but I have no idea how it is supposed to be done;
I can't call it TTW (should I try to publish the methods?!).
I tried it in a debug session:
$ bin/instance debug
>>> portal = app.Plone
>>> from Products.myproduct.exportimport.catalog import exportCatalogTool
>>> exportCatalogTool(portal)
Traceback (most recent call last):
File "<console>", line 1, in <module>
File ".../Products/myproduct/exportimport/catalog.py", line 58, in exportCatalogTool
site = context.getSite()
AttributeError: getSite
So, if this is the way to go, it looks like I need a "real" context.
Update: To get this context, I tried an External Method:
# -*- coding: utf-8 -*-
from Products.myproduct.exportimport.catalog import exportCatalogTool
from pdb import set_trace
def p(dt, dd):
print '%-16s%s' % (dt+':', dd)
def main(self):
"""
Export the portal_user_catalog
"""
g = globals()
print '#' * 79
for a in ('__package__', '__module__'):
if a in g:
p(a, g[a])
p('self', self)
set_trace()
exportCatalogTool(self)
However, wenn I called it, I got the same <PloneSite at /Plone> object as the argument to the main function, which didn't have the getSite attribute. Perhaps my site doesn't call such External Methods correctly?
Or would I need to mention this module somehow in my configure.zcml, but how? I searched my directory tree (especially below Products/myproduct/profiles) for exportimport, the module name, and several other strings, but I couldn't find anything; perhaps there has been an integration once but was broken ...
So how do I make this portal_user_catalog work?
Thank you!
Update: Another debug session suggests the source of the problem to be some transaction matter:
>>> portal = app.Plone
>>> puc = portal.portal_user_catalog
>>> puc._catalog()
[]
>>> profiles_folder = portal.some_folder_with_profiles
>>> for o in profiles_folder.objectValues():
... puc.catalog_object(o)
...
>>> puc._catalog()
[<Products.ZCatalog.Catalog.mybrains object at 0x69ff8d8>, ...]
This population of the portal_user_catalog doesn't persist; after termination of the debug session and starting fg, the brains are gone.
It looks like the problem was indeed related with transactions.
I had
import transaction
...
class Browser(BrowserView):
...
def processNewUser(self):
....
transaction.commit()
before, but apparently this was not good enough (and/or perhaps not done correctly).
Now I start the transaction explicitly with transaction.begin(), save intermediate results with transaction.savepoint(), abort the transaction explicitly with transaction.abort() in case of errors (try / except), and have exactly one transaction.commit() at the end, in the case of success. Everything seems to work.
Of course, Plone still doesn't take this non-standard catalog into account; when I "clear and rebuild" it, it is empty afterwards. But for my application it works well enough.
I created a Qt resource file with all my ui files inside of it, I compiled that with pyrcc4 command-line in a python file, and then I loaded the ui files using loadUi. Here an example:
#!/usr/bin/env python
#-*- coding:utf-8 -*-
import os
import sys
from PyQt4.QtCore import Qt, QFile
from PyQt4.uic import loadUi
from PyQt4.QtGui import QDialog
from xarphus.gui import ui_rc
# I import the compiled qt resource file named ui_rc
BASE_PATH = os.path.dirname(os.path.abspath(__file__))
#UI_PATH = os.path.join(BASE_PATH, 'gui', 'create_user.ui')
UI_PATH = QFile(":/ui_file/create_user.ui")
# I want to load those compiled ui files,
# so I just create QFile.
class CreateUser_Window(QDialog):
def __init__(self, parent):
QDialog.__init__(self, parent)
# I open the created QFile
UI_PATH.open(QFile.ReadOnly)
# I read the QFile and load the ui file
self.ui_create_user = loadUi(UI_PATH, self)
# After then I close it
UI_PATH.close()
Well its works fine, but I have a problem. When I open the GUI-window once, everything works fine. After closing the window I try to open the same GUI-window again, I get ja long traceback.
Traceback (most recent call last): File "D:\Dan\Python\xarphus\xarphus\frm_mdi.py", line 359, in
create_update_form self.update_form = Update_Window(self) File
"D:\Dan\Python\xarphus\xarphus\frm_update.py", line 135, in init
self.ui_update = loadUi(UI_PATH, self) File
"C:\Python27\lib\site-packages\PyQt4\uic__init__.py", line 238, in
loadUi return DynamicUILoader(package).loadUi(uifile, baseinstance,
resource_suffix) File
"C:\Python27\lib\site-packages\PyQt4\uic\Loader\loader.py", line 71,
in loadUi return self.parse(filename, resource_suffix, basedir) File
"C:\Python27\lib\site-packages\PyQt4\uic\uiparser.py", line 984, in
parse document = parse(filename) File
"C:\Python27\lib\xml\etree\ElementTree.py", line 1182, in parse
tree.parse(source, parser) File
"C:\Python27\lib\xml\etree\ElementTree.py", line 657, in parse
self._root = parser.close() File
"C:\Python27\lib\xml\etree\ElementTree.py", line 1654, in close
self._raiseerror(v) File "C:\Python27\lib\xml\etree\ElementTree.py",
line 1506, in _raiseerror raise err xml.etree.ElementTree.ParseError:
no element found: line 1, column 0
Can everyone help me?
Maybe I have a solution, but I don't know if that is a perfectly pythonic.
Well, we know all python projects have an __ init __-file. We need it for initializing Python packages, right? Well I thought: Why not use this file? What did I do? I define in the __ init __ -file a function like so:
#!/usr/bin/env python
#-*- coding:utf-8 -*-
from PyQt4.uic import loadUi
from PyQt4.QtCore import Qt, QFile
def ui_load_about(self):
uiFile = QFile(":/ui_file/about.ui")
uiFile.open(QFile.ReadOnly)
self.ui_about = loadUi(uiFile)
uiFile.close()
return self.ui_about
Now in my "About_Window"-class I do this:
#!/usr/bin/env python
#-*- coding:utf-8 -*-
import os
import sys
from PyQt4.QtCore import Qt, QFile
from PyQt4.uic import loadUi
from PyQt4.QtGui import QDialog
import __init__ as ui_file
class About_Window(QDialog):
def __init__(self, parent):
QDialog.__init__(self, parent)
self.ui_about = ui_file.ui_load_about(self)
You see I importe the package-file (__ init __-file) as ui_file and then I call the function and save the return of the function in the variable self.ui_about.
In my case I open the About_Window from a MainWindow(QMainWindow), and it looks like so:
def create_about_form(self):
self.ui_about = About_Window(self)
# Now when I try to show (show()-method) a window I get two windows
# The reason is: I open and load the ui files from compiled
# qt resorce file that was define in __init__-module.
# There is a function that opens the resource file, reads
# the ui file an closes and returns the ui file back
# That's the reason why I have commented out this method
#self.ui_about.show()
You see I commented out the show()-method. It works without this method. I only define the About_Window()-class. Well I know that isn't maybe the best solution, but it works. I can open the window again and again without traceback.
If you have a better solution or idea let me know :-)
I have a setup such that an nginx server passes control off to uWsgi, which launches a pylons app using the following in my xml configuration file:
<ini-paste>...</ini-paste>
Everything is working nicely, and I was able to set it to debug mode using the following in the associated ini file, like:
debug = true
Except debug mode only prints out errors, and doesn't reload the code everytime a file has been touched. If I was running directly through paste, I could use the --reload option, but going through uWsgi complicates things.
Does anybody know of a way to tell uWsgi to tell paste to set the --reload option, or to do this directly in the paste .ini file?
I used something like the following code to solve this, the monitorFiles(...) method is called on application initialization, and it monitors the files, sending the TERM signal when it sees a change.
I'd still much prefer a solution using paster's --reload argument, as I imagine this solution has bugs:
import os
import time
import signal
from deepthought.system import deployment
from multiprocessing.process import Process
def monitorFiles():
if deployment.getDeployment().dev and not FileMonitor.isRunning:
monitor = FileMonitor(os.getpid())
try: monitor.start()
except: print "Something went wrong..."
class FileMonitor(Process):
isRunning = False
def __init__(self, masterPid):
self.updates = {}
self.rootDir = deployment.rootDir() + "/src/python"
self.skip = len(self.rootDir)
self.masterPid = masterPid
FileMonitor.isRunning = True
Process.__init__(self)
def run(self):
while True:
self._loop()
time.sleep(5)
def _loop(self):
for root, _, files in os.walk(self.rootDir):
for file in files:
if file.endswith(".py"):
self._monitorFile(root, file)
def _monitorFile(self, root, file):
mtime = os.path.getmtime("%s/%s" % (root, file))
moduleName = "%s/%s" % (root[self.skip+1:], file[:-3])
moduleName = moduleName.replace("/",".")
if not moduleName in self.updates:
self.updates[moduleName] = mtime
elif self.updates[moduleName] < mtime:
print "Change detected in %s" % moduleName
self._restartWorker()
self.updates[moduleName] = mtime
def _restartWorker(self):
os.kill(self.masterPid, signal.SIGTERM)
Use the signal framework in 0.9.7 tree
http://projects.unbit.it/uwsgi/wiki/SignalFramework
An example of auto-reloading:
import uwsgi
uwsgi.register_signal(1, "", uwsgi.reload)
uwsgi.add_file_monitor(1, 'myfile.py')
def application(env, start_response):
...