Little Lwt Port Scanner Not Working - ocaml

I'm trying to write a small Lwt (and batteries) port scanner to better
understand Lwt however I'm getting a strange exception whenever I try to scan
too many ports at a time with scan_ports_range. I'm suspecting my mechanism
for keeping a maximum number of connections open is not working...
Exception: Unix.Unix_error (Batteries.Unix.EMFILE, "socket","").
Fatal error: exception Sys_error("/home/(censored)/.opam/4.00.1/lib/utop: Too many open files")
(Which also crashes utop btw)
Code is below. To trigger the error evaluate:
scan_ports_range ~host:"127.0.0.1" (1000,2000)
Any critique/suggestions for my lwt style is also welcome since I've
only started learning it.
open Lwt
let addr_parts addr =
let (host, port) = String.split addr ":" in (host, int_of_string port)
let addr ~host ~port =
lwt entry = Lwt_unix.gethostbyname host in
if Array.length entry.Unix.h_addr_list = 0 then begin
failwith (Printf.sprintf "no address found for host %S\n" host)
end;
return (Unix.ADDR_INET (entry.Unix.h_addr_list.(0), port))
let test_connection ?(timeout=1.0) addr =
let fd = Lwt_unix.socket Unix.PF_INET Unix.SOCK_STREAM 0 in
let connect_close =
(Lwt_unix.connect fd addr) >>= (fun () -> Lwt_unix.close fd) in
try_lwt
(pick [connect_close ; Lwt_unix.timeout timeout])
>>= (fun () -> return true)
with _ -> return false
let scan_ports ~host ~ports =
ports |> Lwt_list.map_p (fun port ->
lwt adr = addr ~host ~port in
test_connection adr >>= (fun res -> return (res,port)) )
>>= (fun l -> return ( l |> List.filter_map ( function
| false, _ -> None | true, port -> Some(port) ) ) )
let scan_ports_range ?(max_open=20) ~host (a, b) =
let rec loop acc enum =
match Enum.peek enum with
| None -> acc |> List.concat |> List.rev |> return
| Some _ ->
let ports = enum |> Enum.take max_open |> List.of_enum in
let open_ports = scan_ports ~host ~ports in
open_ports >>= (fun l -> loop (l::acc) enum )
in loop [] (a--b)

As a wild guess, I think you need to force the closing of the socket in case of timeout, so the cullprint might be:
pick [connect_close ; Lwt_unix.timeout timeout]

Here is the corrected code according to Thomas.
let test_connection ?(timeout=1.0) addr =
let fd = Lwt_unix.socket Unix.PF_INET Unix.SOCK_STREAM 0 in
let connect_close =
(Lwt_unix.connect fd addr) >>= (fun () -> Lwt_unix.close fd) in
try_lwt
(pick [connect_close ; Lwt_unix.timeout timeout])
>>= (fun () -> return true)
with _ -> (Lwt_unix.close fd) >>= (fun () -> return false)

Related

This expression has type H2_lwt_unix.Client.t but an expression was expected of type 'weak702 Lwt.t

How to deal with 'a Lwt objects in a function?
My code is
Array.map (fun conn -> let* resp = (call_server conn
(RequestVoteArg({
candidateNumber = myState.myPersistentState.id;
term = myState.myPersistentState.currentTerm;
lastlogIndex = (Array.get myState.myPersistentState.logs ((Array.length myState.myPersistentState.logs) - 1)).index;
lastlogTerm = (Array.get myState.myPersistentState.logs ((Array.length myState.myPersistentState.logs) - 1)).term
}))) in (match resp with
| Error(s) -> Printf.printf "requestVote: connection failed: %s" s
| Ok(repl, s) ->
(match repl with
| RequestVoteRet(repl) ->
if repl.voteGranted then current_vote := !current_vote + 1;
if not (repl.term = (-1l)) then myState.myPersistentState.currentTerm <- repl.term;
Printf.printf "requestVote: status: %s" s
| _ -> failwith "Should not reach here")); conn) peers
But there is an error: This expression (conn in the end) has type H2_lwt_unix.Client.t but an expression was expected of type 'weak702 Lwt.t.
peers is an array of connections (type: H2_lwt_unix.Client.t).
The definition of call_server is:
val call_server: H2_lwt_unix.Client.t -> protobufArg -> (Types.protobufRet * Grpc.Status.t, Grpc.Status.t) result Lwt.t
and
let build_connection addr port =
let* addrs =
Lwt_unix.getaddrinfo addr (string_of_int port)
[ Unix.(AI_FAMILY PF_INET) ]
in
let socket = Lwt_unix.socket Unix.PF_INET Unix.SOCK_STREAM 0 in
let* () = Lwt_unix.connect socket (List.hd addrs).Unix.ai_addr in
let error_handler _ = print_endline "error" in
let connection =
H2_lwt_unix.Client.create_connection ~error_handler socket
in connection
let call_server connection req =
let enc = Pbrt.Encoder.create() in
match req with
| RequestVoteArg(s) ->
let proto_s =
Proto.Proto_types.default_request_vote_arg ~candidate_number:s.candidateNumber ~term:s.term ~lastlog_term:s.lastlogTerm ~lastlog_index:s.lastlogIndex ()
in
Proto.Proto_pb.encode_request_vote_arg proto_s enc;
Client.call ~service:"raft.Proto" ~rpc:"RequestVote"
~do_request:(H2_lwt_unix.Client.request connection ~error_handler:ignore)
~handler:
(Client.Rpc.unary (Pbrt.Encoder.to_string enc) ~f:(fun decoder ->
(let+ decoder = decoder in
(match decoder with
| Some (decoder) ->
let decoder = Pbrt.Decoder.of_string decoder in
let reply = Proto.Proto_pb.decode_request_vote_reply decoder in
RequestVoteRet( { term=reply.term; voteGranted=reply.vote_granted } )
| None -> RequestVoteRet( { term=(-1l); voteGranted=false } ))))) ()
| AppendEntriesArg(s) ->
let proto_s =
Proto.Proto_types.default_append_entries_arg ~term: s.term ~leader_id:s.leaderID ~next_log_index:s.nextLogIndex ~next_log_term:s.nextLogTerm ~entries:(List.map (fun lg -> Proto.Proto_types.default_log ~command: lg.command ~term:lg.term ~index: lg.index ()) (Array.to_list (s.entries)) ) ()
in
Proto.Proto_pb.encode_append_entries_arg proto_s enc;
Client.call ~service:"raft.Proto" ~rpc:"AppendEntries"
~do_request:(H2_lwt_unix.Client.request connection ~error_handler:ignore)
~handler:
(Client.Rpc.unary (Pbrt.Encoder.to_string enc) ~f:(fun decoder ->
let+ decoder = decoder in
match decoder with
| Some(decoder) ->
let decoder = Pbrt.Decoder.of_string decoder in
let reply = Proto.Proto_pb.decode_append_entries_reply decoder in
AppendEntriesRet( { term=reply.term; success=reply.success } )
| None -> AppendEntriesRet( { term=(-1l); success=false } ))) ()
Your code can be simplified to:
let f (conn:H2_lwt_unix.Client.t) =
let* resp = ... in
...; conn
which fails because conn is a connection and not a promise of returning a connection.
The easiest fix is to wrap this value inside a Lwt promise with either
let f (conn:H2_lwt_unix.Client.t) =
let* resp = ... in
...; Lwt.return conn
or
let f (conn:H2_lwt_unix.Client.t) =
let+ resp = ... in
...; conn

Is this use of Obj.magic necessary?

I am reading a repository and I encountered this function in the body of some Yojson json parsing code:
let load_problems channel =
let open Yojson.Basic.Util in
let j = Yojson.Basic.from_channel channel in
...
let rec unpack x =
try magical (x |> to_int) with _ ->
try magical (x |> to_float) with _ ->
try magical (x |> to_bool) with _ ->
try
let v = x |> to_string in
if String.length v = 1 then magical v.[0] else magical v
with _ ->
try
x |> to_list |> List.map ~f:unpack |> magical
with _ -> raise (Failure "could not unpack")
in
...
where magical = Obj.magic. I understand what Obj.magic is (it's the equivalent to Unsafe.Coerce in Haskell), but I don't see why a type coercion is necessary here. The Yojson.Basic.Util functions the author uses should already either succeed or fail to do this conversion. Any intuition?
EDIT:
I feel I was depriving #glennsl of context, so here is the immediately following passage in which unpack is used:
let tf = j |> member "tasks" |> to_list |> List.map ~f:(fun j ->
let e = j |> member "examples" |> to_list in
let task_type = j |> member "request" |> deserialize_type in
let examples = e |> List.map ~f:(fun ex -> (ex |> member "inputs" |> to_list |> List.map ~f:unpack,
ex |> member "output" |> unpack)) in
let maximum_frontier = j |> member "maximumFrontier" |> to_int in
let name = j |> member "name" |> to_string in
let task =
(try
let special = j |> member "specialTask" |> to_string in
match special |> Hashtbl.find task_handler with
| Some(handler) -> handler (j |> member "extras")
| None -> (Printf.eprintf " (ocaml) FATAL: Could not find handler for %s\n" special;
exit 1)
with _ -> supervised_task) ~timeout:timeout name task_type examples
in
(task, maximum_frontier))
in
There are a number of different task_handlers, but the one I happen to be concerned with is defined as follows:
(fun extras ?timeout:(timeout = 0.001) name ty examples ->
let open Yojson.Basic.Util in
let cost_matters =
try
extras |> member "costMatters" |> to_bool
with _ -> assert false
in
let by = match examples with
| [([0],y)] ->
Bigarray.(Array1.of_array int8_unsigned c_layout (Array.of_list y))
| [([1],y)] ->
Bigarray.(Array1.of_array int8_unsigned c_layout (Array.of_list y))
| _ -> failwith "not a turtle task" in
{ name = name ;
task_type = ty ;
log_likelihood =
(fun p ->
try
match run_recent_logo ~timeout p with
| Some(bx,cost) when (LogoLib.LogoInterpreter.fp_equal bx by 0) ->
(if cost_matters then (0.-.cost)*.10. else 0.)
| _ -> log 0.
with (* We have to be a bit careful with exceptions if the
* synthesized program generated an exception, then we just
* terminate w/ false but if the enumeration timeout was
* triggered during program evaluation, we need to pass the
* exception on
*)
| UnknownPrimitive(n) -> raise (Failure ("Unknown primitive: "^n))
| EnumerationTimeout -> raise EnumerationTimeout
| _ -> log 0.0)
});;
The author also uses ;; in a lot of files..another quirk.

Creating a Chat Server

I am creating a chat server in the Ocaml language using lwt. What I want it to be able to do is prompt the user to enter a nickname, store the nickname, and then output that the user has joined the chat.
I have already tried implementing my own code, as shown below.
let sessions = ref [("",Lwt_io.null)]
(* + obtain initial nick(name),
+ add (nick,outp) to !sessions, and
+ announce join to other users *)
let handle_login nr (inp,outp) =
Lwt_io.printl "Enter initial nick:" >>= (* Print a welcome message to outp *)
fun () -> Lwt_io.read_line inp >>= (* Read the user's nick from inp *)
fun s -> Lwt.return (nr := s); (* Assign the new value to nr *)
sessions := List.concat nr (* Adds nr to sessions *)
send_all nr "<joined>" >>= (* Inform the other users that the user joined *)
fun () -> Lwt.return ()
let rec send_all sender msg = Lwt.return ()
let chat_server _ (inp,outp) =
let nick = ref "" in
let _ = handle_login nick in (* Calls handle_login with nick *)
let rec main_loop () =
Lwt_io.read_line inp >>= handle_input nick outp >>= main_loop in
Lwt.catch main_loop (fun e -> handle_error e !nick inp outp)
open Lwt.Infix
let port = if Array.length Sys.argv > 1 then int_of_string (Sys.argv.(1)) else 16384
let s = Lwt_io.establish_server_with_client_address (Unix.ADDR_INET(Unix.inet_addr_any, port)) ChatServer.chat_server
let _ = Lwt_main.run (fst (Lwt.wait ()))
let _ = s >>= Lwt_io.shutdown_server (* never executes; you might want to use it in utop, though *)
The result should be that when running the program, it will:
print a welcome message to outp,
read the nick from inp,
assign the new value to nr
Add nr to sessions
Announce that the user has joined.
Right now, it compiles, but does not output anything.

How to set a timeout for tests with OUnit?

I have some tests on infinite lazy structures that might run indefinitely if the tested function is not correctly implemented, but I can’t find in the OUnit docs how to set a timeout on tests.
If you're using OUnit2, the following should work:
let tests =
"suite" >::: [OUnitTest.TestCase (
OUnitTest.Short,
(fun _ -> assert_equal 2 (1+1))
);
OUnitTest.TestCase (
OUnitTest.Long,
(fun _ -> assert_equal 4 (2+2))
)]
The type test_length is defined as:
type test_length =
| Immediate
| Short
| Long
| Huge
| Custom_length of float
I don't think that oUnit provides this functionality. I remember having to do this a while back and this is the quick hack I've come up with:
let race seconds ~f =
let ch = Event.new_channel () in
let timeout = Thread.create (fun () ->
Thread.delay seconds;
`Time_out |> Event.send ch |> Event.sync
) () in
let tf = Thread.create (fun () ->
`Result (f ()) |> Event.send ch |> Event.sync) () in
let res = ch |> Event.receive |> Event.sync in
try
Thread.kill timeout;
Thread.kill tf;
res
with _ -> res
let () =
let big_sum () =
let arr = Array.init 1_000_000 (fun x -> x) in
Array.fold_left (+) 0 arr in
match race 0.0001 ~f:big_sum with
| `Time_out -> print_endline "time to upgrade";
| `Result x -> Printf.printf "sum is: %d\n" x
This worked well enough for my use case but I'd definitely would not recommend using this if only because race will not work as you'd expect if ~f does no allocations or calls Thread.yield manually.

Generate natural numbers and printing them using 2 communication servers in Concurrent ML

I have a homework where i need to write 2 communication servers, one which generates natural numbers and the other which prints them. the generating server will be sending to the printing server. The servers should communicate over the shared channel chan.The main function should spawn a thread for each server.
`
val sender = fn : int -> unit
val receiver = fn : unit -> 'a
val main = fn : unit -> unit
`
And so far this is code i have written:
`
datatype 'a inflist = NIL
| CONS of 'a * (unit -> 'a inflist);
fun HD (CONS(a,b)) = a
| HD NIL = raise Subscript;
fun TL (CONS(a,b)) = b()
| TL NIL = raise Subscript;
fun NULL NIL = true
| NULL _ = false;
fun TAKE(xs, 0) = []
| TAKE(NIL, n) = raise Subscript
| TAKE(CONS(x,xf), n) = x::TAKE(xf(), n-1);
fun FROMN n = CONS (n,fn () => FROMN (n+1));
val natnumber = FROMN 0;
fun printGenList f (h::t) = (f h; printGenList f t);
fun printList l = printGenList (fn(e) => print(Int.toString(e)^" ")) l;
fun printPairList l = printGenList (fn(e,f) => print("("^Int.toString(e)^", "^Int.toString(f)^") ")) l;
CM.make "$cml/cml.cm";
open CML;
val chan: int chan = channel();
fun gen ch () = send (ch, printList(TAKE(natnumber,101)));
fun printnat ch () = recv (ch);
fun main () =
let
val ch = channel() :int chan ;
val _ = spawn (gen ch);
val _ = spawn (printnat ch);
in
()
end;
`
But i am not getting the output. Am i going wrong in my syntax or the logic?
I am new to SML and Concurrent ML. Please help me.
Why are you using a infinite list? There are simpler ways to implement this.