BorlandTalk.com

CnA · Guest

I have a problem trying to open this website (http://www.peb.gov.sg/peb/process/searchSubmit) directly as it's a search form that does not have a unique address tagged on to it.

However, i had to open the parent site
(http://www.peb.gov.sg/peb/process/searchPe) prior to opening the above link.

Due to this problem i am not able to obtain the html source of this site : http://www.peb.gov.sg/peb/process/searchSubmit and hence does not allow me to retrieve the required data that i want to store into the database of my program.

I would like to find out if there is any command that will allow me to retrieve the html source for the above case.
Thank You.

Remy Lebeau (TeamB) · Guest

"CnA" <meijuntham (AT) yahoo (DOT) com.sg> wrote

CnA · Guest

But the url that contain search submit (http://www.peb.gov.sg/peb/process/searchSubmit?alphabet=A) does not allow us to open this site directly from the internet explorer. (ie have to access via the parent page)

Below is my code from two units(Unit1 and Connection):

===============================================================
===============================================================

unit Unit1;

interface

uses
Windows, Messages, SysUtils, Variants, Classes, Graphics, Controls, Forms,
Dialogs, DB, DBTables, ComCtrls, DBCtrls, Grids, DBGrids, StdCtrls,
ExtCtrls,IdException, IdBaseComponent, IdComponent, IdTCPConnection,
IdTCPClient, IdHTTP, Shellapi, ScktComp;

type
TFormDataRetrieve = class(TForm)
Panel1: TPanel;
Label1: TLabel;
LabelExit: TLabel;
BtnSearch: TButton;
ProgressBar1: TProgressBar;
TableCourse: TTable;
DBGridCourse: TDBGrid;
DataSource1: TDataSource;
IdHTTP1: TIdHTTP;
QueryID: TQuery;
QueryInsert: TQuery;
DBNavigator1: TDBNavigator;
Label2: TLabel;
LabelLink: TLabel;
Label3: TLabel;
Label4: TLabel;
Label5: TLabel;
Label6: TLabel;
Label7: TLabel;
Label8: TLabel;

procedure BtnSearchClick(Sender: TObject);

procedure getHTMLContent(IEAddress: string);
procedure getdataonline();

procedure LabelExitClick(Sender: TObject);
procedure LabelLinkClick(Sender: TObject);
procedure FormCreate(Sender: TObject);

private
{ Private declarations }
public
{ Public declarations }
HTMLContent : String;
RegNo : Array[0..512] of String;
Name : Array[0..512] of String;

end;

var
FormDataRetrieve: TFormDataRetrieve;

implementation

uses Connection;

{$R *.dfm}

procedure TFormDataRetrieve.BtnSearchClick(Sender: TObject);
var
forex, ForexID, sqlCommand : String;
i : integer;
yr, mon, day, hr, min, sec, msec : string;
h,m,s,ms,y,mo,d :word;
begin
BtnSearch.Enabled := false;
ProgressBar1.Visible:= true;

DbGridCourse.Enabled := true;
DBNavigator1.Visible:= false;

if FormInetConnection.ProxyAddress ='' then
begin
FormInetConnection.Show;
FormInetConnection.SetFocus;
end

else
begin
if FormInetConnection.ProxyAddress <>'NA' then
begin
IdHTTP1.Request.ProxyServer := FormInetConnection.ProxyAddress;
IdHttp1.Request.ProxyPort := FormInetConnection.ProxyPort;
end;

getdataonline();

//Update the value retrieve online to table
QueryInsert.Close;
QueryInsert.DatabaseName:= TableCourse.DatabaseName;

i := 0;
TableCourse.Open;
TableCourse.First;
while not (i = 512) and (RegNo[i] <> '') do
begin
TableCourse.Edit;
TableCourse.FieldByName('REG_NO').AsFloat := StrtoFloat(RegNo[i]);
TableCourse.Post;
TableCourse.FieldByName('Name').AsString := Name[i];
TableCourse.Post;
TableCourse.Close;
i := i + 1;

end;

MessageDlg('Online Data has been completed successfully!', mtInformation, [mbOk] , 0);
IDHttp1.DisconnectSocket;
IDHttp1.Disconnect;
end;

DbGridCourse.Enabled := true;
ProgressBar1.Visible:= false;
DBNavigator1.Visible:= true;
BtnSearch.Enabled := true;
TableCourse.Refresh;

end;

procedure TFormDataRetrieve.getHTMLContent(IEAddress: string);
begin
TRY
HTMLContent := IdHTTP1.Get(IEAddress);
EXCEPT
on EIdConnClosedGracefully do
begin
TRY
//on connection close error, try to connect 1 more time
HTMLContent := IdHTTP1.Get(IEAddress);
Except
begin
//MessageDlg('Connection Error ! Please Check the Internet Server connection in Internet Tools', mtWarning, [mbOK],0);
//abort;
Exception.create('Error happened !');
end;

End;
end;
else
begin
// MessageDlg('Connection Error ! Please Check the Internet Server connection in Internet Tools', mtWarning, [mbOK],0);
//abort;
Exception.create('Error happened !');
end;
END;
end;

//------------------------ Search Online Data ------------------------
procedure TFormDataRetrieve.getDataOnline();
var
IEAddress, OnlineID, Data, BidPrice, AskPrice : String;
CharLeft : integer;
CursorPos :integer;
CursorPos1 : integer;
i,err : integer;
isFloat : real;
IndexNo, CountIndex : Integer;
TempData : String;
forex, ForexID : String;
begin

//Set the IEAddress (go to www.peb.gov.sg)
IEAddress := 'http://www.peb.gov.sg/peb/process/searchSubmit';

//get the HTMLContent and read the webpage
HTMLContent :='';
getHTMLContent(IEAddress);

//-------------------------------- Get Data for Database -------------------

//Find the Required Data
Data := HTMLContent;
i := 0;
while CursorPos <= 0 do
begin
OnlineID := 'Electrical';
CursorPos := pos(OnlineID,Data);
CursorPos1 := CursorPos;

//Proceed, if any value found
CharLeft := 500;
if CursorPos1 > 0 then
begin
Data := Copy(HTMLContent,CursorPos1 - 500,CharLeft);

//Locate the decimal number(price) after a '='
IndexNo := 1;
CountIndex := 1;
While RegNo[i] = '' Do
begin
cursorpos1 := pos('=', Data);
if CursorPos1 > 0 then
begin
TempData := copy(Data, CursorPos1 + 1, CharLeft);

if TempData[IndexNo] in ['0'..'9'] then
begin
While TempData[IndexNo] in ['0'..'9'] do
begin
CountIndex := CountIndex + 1;
IndexNo := IndexNo + 1;
end;
RegNo[i] := Copy(Data, CursorPos1 + 1, CountIndex-1)
end
else
Data := Copy(Data,CursorPos1 + 1, CharLeft);
end;
end;

Data := Copy(Data,CursorPos1 + CountIndex, CharLeft); //Trim off the value found

end;

IndexNo := 1;
CountIndex := 1;
While AskPrice = '' Do
begin
cursorpos1 := pos('>', Data);
if CursorPos1 > 0 then
begin
TempData := copy(Data, CursorPos1 + 1, CharLeft);

if TempData[IndexNo] <> '<' then
begin
While TempData[IndexNo] <> '<' do
begin
CountIndex := CountIndex + 1;
IndexNo := IndexNo + 1;
end;
Name[i] := Copy(Data, CursorPos1 + 1, CountIndex-1)
end
else
Data := Copy(Data,CursorPos1 + 1, CharLeft);
end;
end;
Data := Copy(HTMLContent,CursorPos,10000);
CursorPos := pos(OnlineID,Data);
i := i + 1;
// QueryID.Next;

end;
end;
===============================================================
===============================================================

unit Connection;

interface

uses
Windows, Messages, SysUtils, Variants, Classes, Graphics, Controls, Forms,
Dialogs, StdCtrls, ExtCtrls,IdException, IdBaseComponent, IdComponent,
IdTCPConnection, IdTCPClient, IdHTTP, ComCtrls, Registry, WinInet, jpeg;

type
TFormInetConnection = class(TForm)
IdHTTP1: TIdHTTP;
Panel1: TPanel;
Label2: TLabel;
RadAuto: TRadioButton;
RadMan: TRadioButton;
BtnProceed: TButton;
BtnCancel: TButton;
ProgressBar1: TProgressBar;
Image3: TImage;
Image2: TImage;
Panel2: TPanel;
Label1: TLabel;
LabelPort: TLabel;
LabelAddress: TLabel;
Image1: TImage;
RadProxy: TRadioButton;
RadGeneral: TRadioButton;
ProxyName2: TEdit;
EditPort: TEdit;
ProxyName: TComboBox;
Label3: TLabel;

procedure RadGeneralClick(Sender: TObject);
procedure RadProxyClick(Sender: TObject);
procedure BtnProceedClick(Sender: TObject);
procedure BtnCancelClick(Sender: TObject);
procedure CheckConn(PName : string; Pport : integer);
procedure FormCreate(Sender: TObject);
procedure AutoSet();
procedure ManualSet();
procedure RadAutoClick(Sender: TObject);
procedure RadManClick(Sender: TObject);
procedure IdHTTP1WorkBegin(Sender: TObject; AWorkMode: TWorkMode;
const AWorkCountMax: Integer);
procedure IdHTTP1Work(Sender: TObject; AWorkMode: TWorkMode;
const AWorkCount: Integer);

private
{ Private declarations }
public
{ Public declarations }
ProxyAddress : string;
ProxyPort : integer;
end;

var
FormInetConnection: TFormInetConnection;
FConnection: Integer;

implementation

uses Unit1;

{$R *.dfm}

procedure TFormInetConnection.FormCreate(Sender: TObject);
var
TxtFile : TextFile;
DirNow, LineRead : String;
i : integer;

begin
FormInetConnection.Position := poMainFormCenter;

DirNow := Copy(GetCurrentDir,1,1)+':mj_search_data';

AssignFile(TxtFile, DirNow + 'Proxy.txt');
Reset(TxtFile);

While not EOF(TxtFile) do
begin
ReadLn(TxtFile, LineRead);
if Trim(LineRead)<>'' then ProxyName.Items.Add(Trim(LineRead));
end;

if ProxyName.Items.Count >0 then ProxyName.ItemIndex:=0;
end;

procedure TFormInetConnection.BtnCancelClick(Sender: TObject);
begin
close;
end;

procedure TFormInetConnection.RadAutoClick(Sender: TObject);
begin
Panel2.Visible := false;
end;

procedure TFormInetConnection.RadManClick(Sender: TObject);
begin
Panel2.Visible := true;
end;

procedure TFormInetConnection.RadGeneralClick(Sender: TObject);
begin
ProxyName.Visible:= false;
LabelPort.Visible:= false;
EditPort.Visible:= false;
LabelAddress.Visible:= false;
end;

procedure TFormInetConnection.RadProxyClick(Sender: TObject);
begin
ProxyName.ItemIndex := 0;
ProxyName.Visible:= true;
LabelPort.Visible:= true;
EditPort.Text:= '8080';
EditPort.Visible:= true;
LabelAddress.Visible:= true;
end;

Function ISConnect : Boolean;
var lFlags : Word;
begin
Result:=False;
If InternetGetConnectedState(@lFlags,0) then Result:=True;
end;

//------------------------- Check internet connection --------------------------
procedure TFormInetConnection.CheckConn(PName : string; PPort: integer);
var
HTMLContent : string;
HTMLContent1 : string;
lResult : Integer;
//const ConnectionName='mysingtel'; //Name of dial up connection
begin
If Not ISConnect then //check if connect
begin
lResult:=InternetAttemptConnect(0);
// lResult:=InternetDial(GetActiveWindow,ConnectionName,INTERNET_DIAL_UNATTENDED,@FConnection,0);
if lResult<>0 then
ShowMessage('Connection failed, Error: '+SysErrorMessage(lResult));
//else MessageDlg('IE Connection OK !' + #10 + 'Please click the search button again !', mtInformation, [mbOK],0);
end;
//When close to proceed, check the internet connection
if PName <>'NA' then
begin
IdHTTP1.Request.ProxyServer := PName;
IdHTTP1.Request.ProxyPort := PPort;

end;

TRY
HTMLContent1 := IdHTTP1.Get('http://www.peb.gov.sg/peb/process/searchPe');
HTMLContent := IdHTTP1.Get('http://www.peb.gov.sg/peb/process/searchSubmit');

MessageDlg('IE Connection OK !' + #10 + 'Please click the search button again !', mtInformation, [mbOK],0);
EXCEPT
on EIdConnClosedGracefully do
begin
TRY
//on connection close error, try to connect 1 more time
HTMLContent := IdHTTP1.Get('http://www.yahoo.com');

MessageDlg('IE Connection OK !' + #10 + 'Please click the search button again !', mtInformation, [mbOK],0);
Except
MessageDlg('Internet connection error !' +#10 +'Please check your connection in Internet options.',mtwarning,[mbOK],0);
ProxyAddress :='';

//on error, disable all the online button
close;
End;
end
else
begin
MessageDlg('Internet connection error !' +#10 +'Please check your connection in Internet options.',mtwarning,[mbOK],0);
ProxyAddress :='';

//on error, disable all the online button
//FormForexPrice.BtnSearch.Enabled := false;
close;
end;

END;
IdHTTP1.DisconnectSocket;
IdHTTP1.Disconnect;

end;

procedure TFormInetConnection.BtnProceedClick(Sender: TObject);
begin
ProgressBar1.Visible := true;
ProgressBar1.Position :=0;
BtnProceed.Enabled := false;

if RadMan.Checked = true then
begin
//IF MANUAL SETTING SELECTED
ManualSet();
end

ELSE //CALL THE AUTO SETTING
begin
AutoSet();
end;

BtnProceed.Enabled:= true;
end;

procedure TFormInetConnection.ManualSet();
var
isInt,err : integer;
begin
//Check the proxyname and port if proxy connection selected
if RadProxy.Checked = true then
begin
if(ProxyName.Text <>'') and (EditPort.Text<>'') then
begin
//Check the port number
val(EditPort.Text,isInt,err);
if err <> 0 then MessageDlg('The proxy port is not valid !', mtWarning, [mbOK],0)
Else
begin
//proceed with proxy connection
{**}CheckConn(Trim(ProxyName.Text),StrToInt(Trim(EditPort.Text)));
ProxyAddress := Trim(ProxyName.Text);
ProxyPort := StrToInt(Trim(EditPort.Text));
close;
end;
end
Else MessageDlg('Please specify the Proxy Address and the port !', mtWarning, [mbOK],0);
end

else
begin
//proceed with general connection
try
{**}CheckConn('NA',0);
ProxyAddress := 'NA';
close;
except
end;
end;
end;

procedure TFormInetConnection.AutoSet();
var
lReg: TRegistry;
ProxyRead, ProxAdd : string;
ProxPort, pos1, isInt : integer;

begin
TRY
//Reading the registery file...
lReg := TRegistry.Create;
lReg.RootKey := HKEY_CURRENT_USER;
lReg.OpenKey('SoftwareMicrosoftWindowsCurrentVersionInternet Settings',false);
ProxyRead := lReg.ReadString('ProxyServer');

if ProxyRead <>'' then
begin
pos1 := pos(':',ProxyRead);

if pos1 > 0 then
begin
ProxAdd := Trim(Copy(ProxyRead,1,pos1-1));
ProxPort := StrToInt(Copy(ProxyRead,pos1+1, length(Proxyread)-pos1));
end;
end;

//setting the indyHttp with proxy
if ProxAdd <>'' then
begin
try
//proceed with proxy connection
{**}CheckConn(ProxAdd,ProxPort);
ProxyAddress := ProxAdd;
ProxyPort := ProxPort;
ProgressBar1.Visible := false;
close;
except
end;
end

//OR setting without Proxy
Else
begin
//proceed with general connection
try
{**}CheckConn('NA',0);
ProxyAddress := 'NA';
ProgressBar1.Visible := false;
close;
except
end;
end;
close;

EXCEPT //any error, show the manual procedure
RadMan.Checked := true;
Panel2.Visible := true;
RadAuto.Enabled := false;
ProgressBar1.Visible := false;
abort;
END;
end;

procedure TFormInetConnection.IdHTTP1WorkBegin(Sender: TObject;
AWorkMode: TWorkMode; const AWorkCountMax: Integer);
begin
ProgressBar1.Max:=AWorkCountMax;
end;

procedure TFormInetConnection.IdHTTP1Work(Sender: TObject;
AWorkMode: TWorkMode; const AWorkCount: Integer);
begin

ProgressBar1.Position:=AWorkCount;
Application.ProcessMessages;

end;

end.

===============================================================

Please advise me on how to write the code to retrieve the html content to be retrieved and feel free to give comments on the above code that is written.

The data to be from the html source are a list of RegNo and Name of all the Professional Engineers under Electrical Dept.
Thank You.

"Remy Lebeau (TeamB)" <no.spam (AT) no (DOT) spam.com> wrote:

Remy Lebeau (TeamB) · Guest

"CnA" <donaldaisy2406 (AT) hotmail (DOT) com.sg> wrote

CnA · Guest

Hi can you be a little more specific where i should include the HTTP_REFERER header in my code and the Post() method.

I can't find the TIdCookieManager and how do you go about attaching it to TIdHTTP?

"Remy Lebeau (TeamB)" <no.spam (AT) no (DOT) spam.com> wrote:

CnA · Guest

Anyway, am i correct to write the code in this manner? (see as follows)

/------------------------ Search Online Data ------------------------
procedure TFormDataRetrieve.getDataOnline();
var
IEAddress, OnlineID, Data, BidPrice, AskPrice : String;
CharLeft : integer;
CursorPos :integer;
CursorPos1 : integer;
i,err : integer;
isFloat : real;
IndexNo, CountIndex : Integer;
TempData : String;
forex, ForexID : String;
Values: TStrings;

begin
IdHTTP1.Request.Referer := 'http://www.peb.gov.sg/peb/process/searchPe';

Values := TStringList.Create;
try
Values['engrg_name'] := '';
Values['co_name'] := '';
Values['field_engrg'] := 'ELECTRICAL';
HTMLContent :=
IdHTTP1.Post('http://www.peb.gov.sg/peb/process/searchSubmit', Values);
finally
Values.Free;
end;

//Set the IEAddress (go to www.peb.gov.sg)
IEAddress := 'http://www.peb.gov.sg/peb/process/searchSubmit';

//get the HTMLContent and read the webpage
HTMLContent :='';
getHTMLContent(IEAddress);

"CnA" <donaldaisy2406 (AT) hotmail (DOT) com> wrote:

Remy Lebeau (TeamB) · Guest

"CnA" <donaldaisy2406 (AT) hotmail (DOT) com> wrote

Remy Lebeau (TeamB) · Guest

"CnA" <donaldaisy2406 (AT) hotmail (DOT) com> wrote

CnA · Guest

I'm using version 6 of delphi program and apparently i cannot find TIdCookieManager on the "Indy Misc" tab of the Component Palette and hence i could not activate the event, CookieManager property, from TIdHTTP. Please help this novice programmer.

"Remy Lebeau (TeamB)" <no.spam (AT) no (DOT) spam.com> wrote:

Don Siders · Guest

CnA · Guest

I haven't any idea what version of Indy i'm using. How can i check for that? And how can i update my version of Indy in any case? Thanks.

"Don Siders" <nanook (AT) joesgarage (DOT) org> wrote:

Don Siders · Guest

CnA · Guest

Hi I'm experiencing some problem in updating my version of Indy. Apparently, the downloads does not automatically update the lib etc folders into delphi. Have u any idea if there's is any help file from the website that will guide me through the downloading of Indy?

"Don Siders" <nanook (AT) joesgarage (DOT) org> wrote: