winapi - c++ picking function is not working -

i have picking function doesn't seems work. function should return true if collide object return 0; , never change.
here picking function

bool d3ddevice::picking(hwnd hwnd, lpdirect3ddevice9 d3ddev, cxfileentity *entity) {  d3dxvector3 v;  d3dxmatrix matproj;  point pt;  d3dviewport9 vp;  getcursorpos(&pt);  screentoclient(hwnd, &pt);  d3ddev->gettransform(d3dts_projection, &matproj);  d3ddev->getviewport(&vp);  v.x =  ( ( ( 2.0f * pt.x ) / vp.height  ) - 1 ) / matproj._11;  v.y = -( ( ( 2.0f * pt.x ) / vp.width ) - 1 ) / matproj._22;  v.z =  1.0f;   d3dxmatrix m;  d3dxvector3 rayorigin,raydir;  d3dxmatrix matview;  d3ddev->gettransform(d3dts_view, &matview);   d3dxmatrixinverse( &m, null, &matview );   // transform screen space pick ray 3d space  raydir.x  = v.x*m._11 + v.y*m._21 + v.z*m._31;  raydir.y  = v.x*m._12 + v.y*m._22 + v.z*m._32;  raydir.z  = v.x*m._13 + v.y*m._23 + v.z*m._33;  rayorigin.x = m._41;  rayorigin.y = m._42;  rayorigin.z = m._43;   // use inverse of matrix  d3dxmatrix matinverse, matworld;  d3ddev->gettransform(d3dts_world, &matworld);   // use inverse of matrix  d3dxmatrixinverse(&matinverse,null,&matworld);   // transform ray origin , direction inv matrix  d3dxvector3 rayobjorigin,rayobjdirection, raydirection;   d3dxvec3transformcoord(&rayobjorigin,&rayorigin,&matinverse);  d3dxvec3transformnormal(&rayobjdirection,&raydirection,&matinverse);  d3dxvec3normalize(&rayobjdirection,&rayobjdirection);   bool hashit;  float distancetocollision;   d3dxintersect(entity->pdrawmesh, &rayobjorigin, &rayobjdirection, &hashit, null, null, null, &distancetocollision, null, null);   return hashit; } 

note: pdrawmesh mutable lpd3dxmesh not lpd3dxbasemesh make different? update

bool d3ddevice::picking(hwnd hwnd, lpdirect3ddevice9 d3ddev, cxfileentity *entity) {     d3dxvector3 v;     d3dxmatrix matproj;     point pt;     d3dviewport9 vp;     d3dxmatrix matinverse, matworld;     d3dxmatrix m;     d3dxvector3 rayorigin,raydir;     d3dxmatrix matview;     d3dxvector3 rayobjspace;     d3dxvector3 rayobjorigin,rayobjdirection, raydirection;      getcursorpos(&pt);     screentoclient(hwnd, &pt);     d3ddev->gettransform(d3dts_projection, &matproj);     d3ddev->getviewport(&vp);      d3ddev->gettransform(d3dts_view, &matview);      // use inverse of matrix     d3ddev->gettransform(d3dts_world, &matworld);     d3dxvector3 vec3( pt.x, pt.y, 1.0f );     d3dxvec3unproject( &rayobjspace, &vec3, &vp, &matproj, &matview, &matworld );     // transform ray origin , direction inv matrix      d3dxmatrix invworld;     d3dxmatrixinverse( &invworld, null, &matworld );      d3dxvector3 camobjspace;     d3dxvector3 campos(0.0, 0.0, -14.0f);     d3dxvec3transformcoord( &camobjspace, &campos, &invworld );       raydir = rayobjspace - camobjspace;      bool hashit;     float distancetocollision;      if(failed(d3dxintersect(entity->pdrawmesh, &rayobjspace, &raydir, &hashit, null, null, null, &distancetocollision, null, null)))     {         postquitmessage(0);     };      if(hashit==1)     {         postquitmessage(0);     }      return hashit; } 

update 2: doesn't intersect ;/. bool d3ddevice::picking(hwnd hwnd, lpdirect3ddevice9 d3ddev, cxfileentity *entity, int z) { d3dxvector3 v; point pt; d3dviewport9 vp; d3dxmatrix matinverse, matworld, m, matview, matproj;

    getcursorpos(&pt);     screentoclient(hwnd, &pt);     d3ddev->gettransform(d3dts_projection, &matproj);     d3ddev->getviewport(&vp);     d3ddev->gettransform(d3dts_world, &matworld);     d3ddev->gettransform(d3dts_view, &matview);      // use inverse of matrix     d3dxvector3 raypos(pt.x, pt.y,0); // near-plane position     d3dxvector3 raydir(pt.x, pt.x,1); // far-plane position     d3dxvec3unproject(&raypos,&raypos,&vp,&matproj,&matview,&matworld);     d3dxvec3unproject(&raydir,&raydir,&vp,&matproj,&matview,&matworld);     raydir -= raypos; // make direction 2 positions     d3dxvec3normalize(&raydir,&raydir); // don't know if necessary.     // transform ray origin , direction inv matrix      bool hashit;     float distancetocollision;      if(failed(d3dxintersect(entity->pdrawmesh, &raypos, &raydir, &hashit, null, null, null, &distancetocollision, null, null)))     {         postquitmessage(0);     };      if(hashit!=0)         postquitmessage(0);      return hashit; } 

update 3: ok intersect ;/ after changing values in function

d3dxmatrixperspectivefovlh(&matprojection,                                d3dxtoradian(45),    // horizontal field of view                                (float)width / (float)height, // aspect ratio                                0.0f,   // near view-plane                                1.0f);    // far view-plane 

ray picking fun one. need start off converting screen coordinate projection space. directx projection space (for x,y) ranges -1 1 in x , y need following.

float px = (((float)mousex / screen_width) * 2.0f) - 1.0f; float py = -(((float)mousey / screen_height) * 2.0f) - 1.0f; 

not py has minus in front because screen coord go top bottom projection space opposite.

you have position in projection space. need create vector along point. thankfully thats nice , easy too.

float pz = 1.0f; 

now can unproject object space:

d3dxvector3 rayobjspace; d3dxvec3unproject( &rayobjspace, &rayout, &viewport, &projectionmatrix, &viewmatrix, &worldmatrix ); 

finally plug d3dxintersect , ready go :)

edit: realise there bug in explanation above. in fact d3dxvec3unproject can pass in actual screen coordinate x,y z of 1.

so:

d3dxvector3 vec3( mousex, mousey, 1.0f ); d3dxvector3 rayobjspace; d3dxvec3unproject( &rayobjspace, &vec3, &viewport, &projectionmatrix, &viewmatrix, &worldmatrix ); 

much simpler. view port transform applied twice.

that said have looked @ ray vectors have been getting returned? point towards object?

edit 2: alright i'll more specific. have world position of point clicked on can calculate ray direction doing following.

 d3dxmatrix invworld;  d3dxmatrixinverse( &invworld, null, &world );   d3dxvector3 camobjspace;  d3dxvec3transformcoord( &camobjspace, &campos, &invworld );    d3xvector3 raydir = rayobjspace - camobjspace;   bool bhit = false;  float disttohit = true;  hresult hr = d3dxintersect( pmesh, &rayobjspace, &raydir, &bhit, null, null, null, &disttohit, null, null );