Using paramiko to tunnel an MySql port when django starts - django

I am trying to connect to a remote MySql server from my local machine.
I want to run it whenever the DEBUG constant is set to true.
Here's the script:
import select
import SocketServer
import sys
import threading
import paramiko
SSH_PORT = 22
DEFAULT_PORT = 4000
g_verbose = True
class ForwardServer (SocketServer.ThreadingTCPServer):
daemon_threads = True
allow_reuse_address = True
class Handler (SocketServer.BaseRequestHandler):
def handle(self):
try:
chan = self.ssh_transport.open_channel('direct-tcpip',
(self.chain_host, self.chain_port),
self.request.getpeername())
except Exception, e:
verbose('Incoming request to %s:%d failed: %s' % (self.chain_host,
self.chain_port,
repr(e)))
return
if chan is None:
verbose('Incoming request to %s:%d was rejected by the SSH server.' %
(self.chain_host, self.chain_port))
return
verbose('Connected! Tunnel open %r -> %r -> %r' % (self.request.getpeername(),
chan.getpeername(), (self.chain_host, self.chain_port)))
while True:
r, w, x = select.select([self.request, chan], [], [])
if self.request in r:
data = self.request.recv(1024)
if len(data) == 0:
break
chan.send(data)
if chan in r:
data = chan.recv(1024)
if len(data) == 0:
break
self.request.send(data)
chan.close()
self.request.close()
verbose('Tunnel closed from %r' % (self.request.getpeername(),))
def forward_tunnel(local_port, remote_host, remote_port, transport):
# this is a little convoluted, but lets me configure things for the Handler
# object. (SocketServer doesn't give Handlers any way to access the outer
# server normally.)
class SubHander (Handler):
chain_host = remote_host
chain_port = remote_port
ssh_transport = transport
ForwardServer(('', local_port), SubHander).serve_forever()
def verbose(s):
if g_verbose:
print s
HELP = """\
Set up a forward tunnel across an SSH server, using paramiko. A local port
(given with -p) is forwarded across an SSH session to an address:port from
the SSH server. This is similar to the openssh -L option.
"""
def forward():
client = paramiko.SSHClient()
client.load_system_host_keys()
client.set_missing_host_key_policy(paramiko.WarningPolicy())
try:
print 'connecting'
client.connect('*******', username='***', password='****!')
print 'connected'
except Exception, e:
print '*** Failed to connect to %s:%d: %r' % ('*****', 22, e)
sys.exit(1)
try:
forward_tunnel(3306, '127.0.0.1', 3306, client.get_transport())
except SystemExit:
print 'C-c: Port forwarding stopped.'
sys.exit(0)
I have two problems here:
1) I don't know how and when to call my forward function when django raises.
2) When I access django locally and run the script from the console I get the following exception:
exception happened during
processing of request from
('127.0.0.1', 41872) Traceback (most
recent call last): File
"/usr/lib/python2.6/SocketServer.py",
line 558, in process_request_thread
self.finish_request(request, client_address) File
"/usr/lib/python2.6/SocketServer.py",
line 320, in finish_request
self.RequestHandlerClass(request, client_address, self) File
"/usr/lib/python2.6/SocketServer.py",
line 615, in init
self.handle() File "/home/omer/Aptana Studio 3
Workspace/Website/src/ssh_tunnel/tunnel.py",
line 51, in handle
verbose('Tunnel closed from %r' % (self.request.getpeername(),)) File
"", line 1, in getpeername
File "/usr/lib/python2.6/socket.py",
line 165, in _dummy
raise error(EBADF, 'Bad file descriptor') error: [Errno 9] Bad file
descriptor
Was this a bad idea to begin with?
Should I do this manually every time?

I don't think it's a bad idea.
I don't think you need to do it manually.
The exception is a bug in paramiko's forward code sample. This has been addressed by jhalcrow in the pull request here:
https://github.com/paramiko/paramiko/pull/36
This post has some code to do it in a more event driven way, i.e if you wanted to call it via some web event hooks in your django code or the like:
Paramiko SSH Tunnel Shutdown Issue

humm, i didn't try this, but if you are on linux, could you run
ssh -L 3306:localhost:3306 remote.host.ip
through python system call when DEBUG is set?
also if you are on Windows, try putty with port forwarding

Related

Fetchingdata from Cloud Firstore results in OSError when SUBPROCESS.POPEN.COMMUNICATE() method is used

I have sample script that starts by opening a connection to firebase cloud database and retrieving the information from the database using get() method. Script then moves to a infinite loop that calls subprocess.popen.communicate() method.
Output -
1. Data is retrieved successfully from the database .
2. Infinite loop runs successfully for 4 minutes and then terminates with OSError: Bad File Descriptor
import sys, os, time, subprocess
import google.auth.transport.requests
import firebase_admin
from firebase_admin import credentials
from firebase_admin import firestore
import datetime
def firebase_retrieve_data():
cred =firebase_admin.credentials.Certificate('path/to/serviceAccount.json')
app=firebase_admin.initialize_app(cred)
db= firestore.client()
doc_ref = db.collection(u'collection_name').document(u'doc_name')
try:
doc = doc_ref.get()
firebase_admin.delete_app(app)
except Exception as e:
firebase_admin.delete_app(app)
print(e)
print(u'No such document!')
firebase_retrieve_data()
while(True):
x=subprocess.Popen("pwd", stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds= True, shell=True).communicate()
Although both pieces seem independent of each other and OSError is generated due to subprocess.popen.communicate() method, I suspect error is related to get() function that retrieves the data from the database. Reason I say this is because, while the script was running I ran following command to get list of active connections in the background
"sudo lsof -i -n | grep -E "(LISTEN|ESTABLISHED)"
I found out that get() function opens a connection to database that remains open for about 4-5 minutes(I assume thats the timeout for the database connection). As soon as this database connection closes, the script fails with OSError: [Errno 9] Bad file descriptor on line
x=subprocess.Popen("pwd", stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds= True, shell=True).communicate()
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/subprocess.py", line 710, in init
errread, errwrite)
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/subprocess.py", line 1335, in _execute_child
raise child_exception
I suspect that it has something to do with improper closing of database connection but all the firebase examples or python firebase documentation I see has no mention of proper closing of database connection()
Plus, I am failing to understand the reason of how firebase database connection should effect call to subprocess.popen.communciate() method
Used
os.system("pwd")
instead of
subprocess.Popen("pwd", stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds= True, shell=True).communicate()
This Works . However, still don't know the reason why subprocess.Popen.communicate() and os.popen() throw an error

when webdriver can’t find elem,Failed to establish a new connection [duplicate]

I have one question:I want to test "select" and "input".can I write it like the code below:
original code:
12 class Sinaselecttest(unittest.TestCase):
13
14 def setUp(self):
15 binary = FirefoxBinary('/usr/local/firefox/firefox')
16 self.driver = webdriver.Firefox(firefox_binary=binary)
17
18 def test_select_in_sina(self):
19 driver = self.driver
20 driver.get("https://www.sina.com.cn/")
21 try:
22 WebDriverWait(driver,30).until(
23 ec.visibility_of_element_located((By.XPATH,"/html/body/div[9]/div/div[1]/form/div[3]/input"))
24 )
25 finally:
26 driver.quit()
# #测试select功能
27 select=Select(driver.find_element_by_xpath("//*[#id='slt_01']")).select_by_value("微博")
28 element=driver.find_element_by_xpath("/html/body/div[9]/div/div[1]/form/div[3]/input")
29 element.send_keys("杨幂")
30 driver.find_element_by_xpath("/html/body/div[9]/div/div[1]/form/input").click()
31 driver.implicitly_wait(5)
32 def tearDown(self):
33 self.driver.close()
I want to test Selenium "select" function.so I choose sina website to select one option and input text in textarea.then search it .but when I run this test,it has error:
Traceback (most recent call last):
File "test_sina_select.py", line 32, in tearDown
self.driver.close()
File "/usr/lib/python2.7/site-packages/selenium/webdriver/remote/webdriver.py", line 688, in close
self.execute(Command.CLOSE)
File "/usr/lib/python2.7/site-packages/selenium/webdriver/remote/webdriver.py", line 319, in execute
response = self.command_executor.execute(driver_command, params)
File "/usr/lib/python2.7/site-packages/selenium/webdriver/remote/remote_connection.py", line 376, in execute
return self._request(command_info[0], url, body=data)
File "/usr/lib/python2.7/site-packages/selenium/webdriver/remote/remote_connection.py", line 399, in _request
resp = self._conn.request(method, url, body=body, headers=headers)
File "/usr/lib/python2.7/site-packages/urllib3/request.py", line 68, in request
**urlopen_kw)
File "/usr/lib/python2.7/site-packages/urllib3/request.py", line 81, in request_encode_url
return self.urlopen(method, url, **urlopen_kw)
File "/usr/lib/python2.7/site-packages/urllib3/poolmanager.py", line 247, in urlopen
response = conn.urlopen(method, u.request_uri, **kw)
File "/usr/lib/python2.7/site-packages/urllib3/connectionpool.py", line 617, in urlopen
release_conn=release_conn, **response_kw)
File "/usr/lib/python2.7/site-packages/urllib3/connectionpool.py", line 617, in urlopen
release_conn=release_conn, **response_kw)
File "/usr/lib/python2.7/site-packages/urllib3/connectionpool.py", line 617, in urlopen
release_conn=release_conn, **response_kw)
File "/usr/lib/python2.7/site-packages/urllib3/connectionpool.py", line 597, in urlopen
_stacktrace=sys.exc_info()[2])
File "/usr/lib/python2.7/site-packages/urllib3/util/retry.py", line 271, in increment
raise MaxRetryError(_pool, url, error or ResponseError(cause))
MaxRetryError: HTTPConnectionPool(host='127.0.0.1', port=51379): Max retries exceeded with url: /session/2e64d2a1-3c7f-4221-96fe-9d0b1c102195/window (Caused by ProtocolError('Connection aborted.', error(111, 'Connection refused')))
----------------------------------------------------------------------
Ran 1 test in 72.106s
who can tell me why?thanks
This error message...
MaxRetryError: HTTPConnectionPool(host='127.0.0.1', port=51379): Max retries exceeded with url: /session/2e64d2a1-3c7f-4221-96fe-9d0b1c102195/window (Caused by ProtocolError('Connection aborted.', error(111, 'Connection refused')))
...implies that the call to self.driver.close() method failed raising MaxRetryError.
A couple of things:
First and foremost as per the discussion max-retries-exceeded exceptions are confusing the traceback is somewhat misleading. Requests wraps the exception for the users convenience. The original exception is part of the message displayed.
Requests never retries (it sets the retries=0 for urllib3's HTTPConnectionPool), so the error would have been much more canonical without the MaxRetryError and HTTPConnectionPool keywords. So an ideal Traceback would have been:
ConnectionError(<class 'socket.error'>: [Errno 1111] Connection refused)
But again #sigmavirus24 in his comment mentioned ...wrapping these exceptions make for a great API but a poor debugging experience...
Moving forward the plan was to traverse as far downwards as possible to the lowest level exception and use that instead.
Finally this issue was fixed by rewording some exceptions which has nothing to do with the actual connection refused error.
Solution
Even before self.driver.close() within tearDown(self) is invoked, the try{} block within test_select_in_sina(self) includes finally{} where you have invoked driver.quit()which is used to call the /shutdown endpoint and subsequently the web driver & the client instances are destroyed completely closing all the pages/tabs/windows. Hence no more connection exists.
You can find a couple of relevant detailed discussion in:
PhantomJS web driver stays in memory
Selenium : How to stop geckodriver process impacting PC memory, without calling
driver.quit()?
In such a situation when you invoke self.driver.close() the python client is unable to locate any active connection to initiate a clousure. Hence you see the error.
So a simple solution would be to remove the line driver.quit() i.e. remove the finally block.
tl; dr
As per the Release Notes of Selenium 3.14.1:
* Fix ability to set timeout for urllib3 (#6286)
The Merge is: repair urllib3 can't set timeout!
Conclusion
Once you upgrade to Selenium 3.14.1 you will be able to set the timeout and see canonical Tracebacks and would be able to take required action.
References
A couple of relevent references:
Adding max_retries as an argument
Removed the bundled charade and urllib3.
Third party libraries committed verbatim
Just had the same problem. The solution was to change the owner of the folder with a script recursively. In my case the folder had root:root owner:group and I needed to change it to ubuntu:ubuntu.
Solution: sudo chown -R ubuntu:ubuntu /path-to-your-folder
Use Try and catch block to find exceptions
try:
r = requests.get(url)
except requests.exceptions.Timeout:
#Message
except requests.exceptions.TooManyRedirects:
#Message
except requests.exceptions.RequestException as e:
#Message
raise SystemExit(e)

httplib2.ServerNotFoundError: Unable to find the server at www.googleapis.com

When i try to fetch the details of gmail using google-api's (googleapiclient,oauth2client), i am getting below error:
File "<stdin>", line 1, in <module>
File "/usr/local/lib/python2.7/site-packages/google_api_python_client-1.7.8-py2.7.egg/googleapiclient/_helpers.py", line 130, in positional_wrapper
File "/usr/local/lib/python2.7/site-packages/google_api_python_client-1.7.8-py2.7.egg/googleapiclient/discovery.py", line 224, in build
File "/usr/local/lib/python2.7/site-packages/google_api_python_client-1.7.8-py2.7.egg/googleapiclient/discovery.py", line 274, in _retrieve_discovery_doc
File "/usr/local/lib/python2.7/site-packages/httplib2-0.12.1-py2.7.egg/httplib2/__init__.py", line 2135, in request
cachekey,
File "/usr/local/lib/python2.7/site-packages/httplib2-0.12.1-py2.7.egg/httplib2/__init__.py", line 1796, in _request
conn, request_uri, method, body, headers
File "/usr/local/lib/python2.7/site-packages/httplib2-0.12.1-py2.7.egg/httplib2/__init__.py", line 1707, in _conn_request
raise ServerNotFoundError("Unable to find the server at %s" % conn.host)
httplib2.ServerNotFoundError: Unable to find the server at www.googleapis.com
but it is working fine in my pc but not from remote location.
code:
from googleapiclient.discovery import build
from oauth2client.service_account import ServiceAccountCredentials
credentials = ServiceAccountCredentials.from_json_keyfile_name(
"quickstart-1551349397232-e8bcb3368ae1.json", scopes=
['https://www.googleapis.com/auth/admin.directory.group', 'https://www.googleapis.com/auth/admin.directory.user', 'https://www.googleapis.com/auth/admin.directory.domain', 'https://www.googleapis.com/auth/gmail.readonly'])
delegated_credentials = credentials.create_delegated('jango#carbonitedepot.com')
DIRECOTORY = build('admin', 'directory_v1', credentials=delegated_credentials)
try:
results = DIRECOTORY.users().list(customer='my_customer').execute()
users = results.get('users', [])
res = []
for info in users:
print(info)
res.append(info.get("primaryEmail"))
print(res)
except Exception as e:
print(e)
Any help would be much appreciated.
Thanks in advance.
I had the same issue and I searched a lot to get it fixed, turns out I was at fault here.
I don't know from where there was podman installed instead of docker and that caused the problem.
I suggest you check your docker version and make sure the latest is running at the server, otherwise, the library works fine! Please let me know if you still face this issue, would like to dig deep!
The httplib2 is likely scanning through network interface DHCP DNS nameservers (in your registry or visible to docker) and then trying to connect through a stale DNS or possible an IPv6. Patching your socket in the main routine may solve it:
# Monkey patch to force IPv4, since FB seems to hang on IPv6
import socket
old_getaddrinfo = socket.getaddrinfo
def new_getaddrinfo(*args, **kwargs):
responses = old_getaddrinfo(*args, **kwargs)
return [response
for response in responses
if response[0] == socket.AF_INET]
socket.getaddrinfo = new_getaddrinfo

socket.send() is raising AttributeError: 'tuple' object has no attribute 'send'

I was developing a program that individual elements from the list to another machine (sender-server,reciever-client). I am sharing my program below
-------------------------client.py------------------------
import socket # Import socket module
s = socket.socket() # Create a socket object
host = socket.gethostname() # Get local machine name
port = 4444 # Reserve a port for your service.
s.connect((host, port))
s.send("Hi server1")
while True:
a=int(s.recv(1024))
tmp=0
while tmp<=a:
print s.recv(1024)
tmp=tmp+1
----------------------------------server.py------------------------------
import socket # Import socket module
s = socket.socket() # Create a socket object
host = socket.gethostname() # Get local machine name
port = 4444 # Reserve a port for your service.
s.bind((host, port)) # Bind to the port
s.listen(5)
print "Server listening"
while True:
c=s.accept()
b=['a','b']
d=len(b)
a=str(d)
c.send(a)
for i in range(0,len(b)):
tmp=str(b[i])
c.send(tmp)
When I run both server and client, the server raises this:
Traceback (most recent call last):
File "server.py", line 14, in <module>
c.send(a)
AttributeError: 'tuple' object has no attribute 'send'
You'll have to fix the indentation on line 11 of client.py.
socket.accept() returns a tuple (conn, addr) where conn is a socket object. You have to use that object in line 14 to send things. What you're doing is calling send() from the entire tuple which has no method named send and so the AttributeError gets raised. I'd suggest changing line 11 to c = s.accept()[0].

Python script to stop Tomcat service error The specified service does not exist as an installed service

I am using a Python script to stop the Tomcat service. I am getting the error:
error: (1060, 'GetServiceKeyName', 'The specified service does not exist as an installed service.')
The method to stop the service is:
def service_info(action, machine, service):
if action == 'stop':
win32serviceutil.StopService(service, machine)
print '%s stopped successfully' % service
elif action == 'start':
win32serviceutil.StartService(service, machine)
print '%s started successfully' % service
elif action == 'restart':
win32serviceutil.RestartService(service, machine)
print '%s restarted successfully' % service
elif action == 'status':
if win32serviceutil.QueryServiceStatus(service, machine)[1] == 4:
print "%s is running normally" % service
else:
print "%s is *not* running" % service
From my base class unittest i call the method like this:
from Utilities.HelperMethods import recursive_overwrite, rename_file, delete_a_file, stop_a_service, service_info
class BaseTestCase(unittest.TestCase):
#classmethod
def setUpClass(cls):
machine = 'riaz-pc'
service_tomcat = 'Tomcat7.exe'
action_stop = 'stop'
action_start = 'start'
service_info(action_stop, machine, service_tomcat)
service_info(action_start, machine, service_tomcat)
I think I am giving the correct service name of Tomcat. In the command prompt CMD i enter tasklist and I can see Tomcat7.exe
I give Tomcat7.exe as the service name.
Why is it failing?
The full error log is:
Traceback (most recent call last):
File "C:\Webdriver\ClearCore 501\Base\BaseTestCase.py", line 35, in setUpClass
service_info(action_stop, machine, service_tomcat)
File "C:\Webdriver\ClearCore 501\Utilities\HelperMethods.py", line 134, in service_info
win32serviceutil.StopService(service, machine)
File "C:\Python27\lib\site-packages\win32\lib\win32serviceutil.py", line 409, in StopService
return ControlService(serviceName, win32service.SERVICE_CONTROL_STOP, machine)
File "C:\Python27\lib\site-packages\win32\lib\win32serviceutil.py", line 318, in ControlService
hs = SmartOpenService(hscm, serviceName, win32service.SERVICE_ALL_ACCESS)
File "C:\Python27\lib\site-packages\win32\lib\win32serviceutil.py", line 80, in SmartOpenService
name = win32service.GetServiceKeyName(hscm, name)
error: (1060, 'GetServiceKeyName', 'The specified service does not exist as an installed service.')
Thanks,
Riaz
I sense the service name is wrong. Try tomcat7 instead of Tomcat7.exe:
win32serviceutil.StopService('tomcat7', 'riaz-pc')